MasterPoster_wPaper_Code_Photos2 Exhibition Thanks! The Code !"#$%&'()*+ ,%$"'-$..+-&!$/0'!"#$%&'/)"+1&2#.+ !"#$%&'0&%2-& !"#$%&'")&3 !"#$%&'/2"#4')0'/# ,%$"'0-!#4'!"#$%&'0!5/). ,%$"'0-!#4'!"#$%&'!/&+%#$.)&+ !"#$%&'.!6%$0) 1+,'&+0&7##89()*+:)"+;'$/0+&<=;'$/0+&>?@A ''''()*+B)&);'#)%)"0'C'.$)1D)*+E!.+9()*+:)"+@ ''''()*+B)&)'C')11FG.$-H9()*+B)&);'#)%)"0@ ''''()*+B)&)C'62&&+%D$%&37##8IJ9()*+B)&)@ ''''$/0+&B)&)'C'$/0+&<-)/9()*+B)&);'$/0+&<=;'$/0+&>?@ ''''KLM/0+&9$/0+&B)&)@ ''''%+&2%/'()*+B)&);'$/0+&B)&) 1+,'KLM/0+&9$/0+&<-)/@A ''''.+/5&3'C'.+/9$/0+&<-)/@ ''''-$2/&+%'C'N ''''(3!.+'-$2/&+%'O'.+/5&3'P'QA ''''''''!,'$/0+&<-)/R-$2/&+%'S'QT'P'$/0+&<-)/R-$2/&+%T'U'8NNNA ''''''''''''#%!/&9-$2/&+%@ ''''''''''''1!,,'C'$/0+&<-)/R-$2/&+%'S'QT'P'$/0+&<-)/R-$2/&+%T ''''''''''''3).,1!,,'C'1!,,'V'W ''''''''''''/#X!/0+%&9$/0+&<-)/;'-$2/&+%SQ;'$/0+&<-)/R-$2/&+%T'S'3).,1!,,@ ''''''''''''.+/5&3'C'.+/5&3'S'Q ''''''''-$2/&+%'C'-$2/&+%'S'Q ''''''''''''Y/+(M/0+&0'C'$/0+&<-)/RLSQTVW ''''''''''''Y$/0+&<-)/RLSQT'C'/+(M/0+&0 ''''''''''''Y/#X!/0+%&9$/0+&<-)/;'LSQ;'/+(M/0+&0@ ''''/#X0)*+&L&9Z$/0+&E!LX-0*Z;'$/0+&<-)/;'1+.!"!&+%CZ;Z@ ''''%+&2%/'$/0+&<-)/ [[[1+,'$/0+&<-)/9()*+FG.$-H@A ''''()*+FG.$-H'C'/#X)0)%%)49()*+FG.$-H;'1&4#+C/#X\$)&8W@ ''''0-)/?+/5&3'C'QNNNNN ''''0-)/=+")!/!/5'C'.+/9()*+FG.$-H@ ''''-2%%+/&<-)/]$0!&!$/'C'N ''''0-)/B)&)'C'/#X+"#&49N@ ''''#%!/&9[#%+?$$#[@ ''''!,'0-)/=+")!/!/5'U'0-)/?+/5&3A ''''''''-2%%+/&<-)/'C'.!6%$0)X$/0+&X$/0+&^1+&+-&94C()*+FG.$-HR-2%%+/&<-)/]$0!&!$/A9-2%%+/&<-)/]$0!&!$/'S'0-)/?+/5&3@T;'0%CQ_NN;'$/0+&^+/*+.$#+C:$/+;'3$#^.+/5&3CW;'6)-H&%)-HC`%2+;'+/+%54C:$/+;'2/!&0C[0)"#.+0[@ '''''''',$%'L'!/'%)/5+9.+/9-2%%+/&<-)/@@A ''''''''''''-2%%+/&<-)/RLT'C'-2%%+/&<-)/RLT'S'-2%%+/&<-)/]$0!&!$/ ''''''''0-)/B)&)'C'/#X)##+/190-)/B)&);'-2%%+/&<-)/@ ''''''''-2%%+/&<-)/]$0!&!$/'C'-2%%+/&<-)/]$0!&!$/'S'0-)/?+/5&3 ''''''''0-)/=+")!/!/5'C'0-)/=+")!/!/5'P'0-)/?+/5&3 ''''''''#%!/&90-)/=+")!/!/5@ ''''-2%%+/&<-)/'C'.!6%$0)X$/0+&X$/0+&^1+&+-&94C()*+FG.$-HR-2%%+/&<-)/]$0!&!$/A9.+/9()*+FG.$-H@'P'Q@T;'0%CQ_NN;'$/0+&^+/*+.$#+C:$/+;'3$#^.+/5&3CW;'6)-H&%)-HC`%2+;'+/+%54C:$/+;'2/!&0C[0)"#.+0[@ '''',$%'L'!/'%)/5+9.+/9-2%%+/&<-)/@@A ''''''''-2%%+/&<-)/RLT'C'-2%%+/&<-)/RLT'S'-2%%+/&<-)/]$0!&!$/ ''''0-)/B)&)'C'/#X)##+/190-)/B)&);'-2%%+/&<-)/@ ''''/#X0)*+&L&9Z-2%%+/&M/0+&X-0*Z;'0-)/B)&);'1+.!"!&+%CZ;Z@ ''''%+&2%/'0-)/B)&) [[[ 1+,'$/0+&<-)/9()*+FG.$-H;'$/0+&<=;'$/0+&>?@A ''''()*+FG.$-H'C'/#X)0)%%)49()*+FG.$-H;'1&4#+C/#X\$)&8W@ ''''0-)/?+/5&3'C'QW8J_a ''''0-)/=+")!/!/5'C'.+/9()*+FG.$-H@ ''''-2%%+/&<-)/]$0!&!$/'C'N ''''0-)/B)&)'C'/#X+"#&49N@ ''''(3!.+'0-)/=+")!/!/5'U'0-)/?+/5&3A ''''''''-2%%+/&<-)/'C'.!6%$0)X$/0+&X$/0+&^1+&+-&94C()*+FG.$-HR-2%%+/&<-)/]$0!&!$/A9-2%%+/&<-)/]$0!&!$/'S'0-)/?+/5&3@T;'0%C$/0+&<=;'$/0+&^+/*+.$#+C:$/+;'3$#^.+/5&3C$/0+&>?;'6)-H&%)-HC`%2+;'+/+%54C:$/+;'2/!&0C[0)"#.+0[@ ''''''''Y#%!/&9-2%%+/&<-)/@ '''''''',$%'L'!/'%)/5+9.+/9-2%%+/&<-)/@@A ''''''''''''-2%%+/&<-)/RLT'C'-2%%+/&<-)/RLT'S'-2%%+/&<-)/]$0!&!$/ ''''''''0-)/B)&)'C'/#X)##+/190-)/B)&);'-2%%+/&<-)/@ ''''''''-2%%+/&<-)/]$0!&!$/'C'-2%%+/&<-)/]$0!&!$/'S'0-)/?+/5&3 ''''''''0-)/=+")!/!/5'C'0-)/=+")!/!/5'P'0-)/?+/5&3 ''''''''Y#%!/&9[-2%%+/&<-)/]$0!&!$/['S'0&%9-2%%+/&<-)/]$0!&!$/@@ ''''''''Y#%!/&9[0-)/=+")!/!/5['S'0&%90-)/=+")!/!/5@@ ''''-2%%+/&<-)/'C'.!6%$0)X$/0+&X$/0+&^1+&+-&94C()*+FG.$-HR-2%%+/&<-)/]$0!&!$/A9.+/9()*+FG.$-H@'P'Q@T;'0%C$/0+&<=;'$/0+&^+/*+.$#+C:$/+;'3$#^.+/5&3C$/0+&>?;'6)-H&%)-HC`%2+;'+/+%54C:$/+;'2/!&0C[0)"#.+0[@ '''',$%'L'!/'%)/5+9.+/9-2%%+/&<-)/@@A ''''''''-2%%+/&<-)/RLT'C'-2%%+/&<-)/RLT'S'-2%%+/&<-)/]$0!&!$/ ''''0-)/B)&)'C'/#X)##+/190-)/B)&);'-2%%+/&<-)/@ ''''/#X0)*+&L&9Z-2%%+/&M/0+&X-0*Z;'0-)/B)&);'1+.!"!&+%CZ;Z@ ''''%+&2%/'0-)/B)&) 1+,')11FG.$-H9()*+B)&);'#)%)"0@A ''''2D)*+'C'&+-3b)%!0#++19()*+B)&);'#)%)"0;'8IJNNN@ ''''-.$-HE!.+;'-.$-H])%)"0'C'.$)1D)*+E!.+9[-.$-HccWX()*[@ ''''-.$-H?+/5&3'C'.+/9-.$-HE!.+@ ''''()*+=+")!/!/5'C'.+/92D)*+@ ''''-2%%+/&]$0!&!$/'C'N ''''-.$-H+1E!.+'C'RT ''''(3!.+'()*+=+")!/!/5'U'-.$-H?+/5&3A ''''''''2G.$-Hd!L'C')21!$d!L92D)*+R-2%%+/&]$0!&!$/A9-2%%+/&]$0!&!$/S-.$-H?+/5&3@T;'X_;'-.$-HE!.+;'X_@ '''''''',$%'L'!/'%)/5+9.+/92G.$-Hd!L@@A ''''''''''''-.$-H+1E!.+X)##+/192G.$-Hd!LRLT@ ''''''''()*+=+")!/!/5'C'()*+=+")!/!/5'P'-.$-H?+/5&3 ''''''''-2%%+/&]$0!&!$/'C'-2%%+/&]$0!&!$/'S'-.$-H?+/5&3 ''''2G.$-Hd!L'C')21!$d!L92D)*+R-2%%+/&]$0!&!$/A.+/92D)*+@T;'X_;'-.$-HE!.+RNA()*+=+")!/!/5T;'X_@ '''',$%'L'!/'%)/5+9.+/92G.$-Hd!L@@A ''''''''-.$-H+1E!.+X)##+/192G.$-Hd!LRLT@ ''''%+&2%/'-.$-H+1E!.+ 1+,'+L)"#.+D)*+d)H+%9!/#2&E%+e@A ''''Y'3&&#AVV0&)-H$*+%\$(X-$"Ve2+0&!$/0V8a8f8_NV3$(P&$P(%!&+P0&+%+$P()*PK.+0P!/P#4&3$/ ''''Y'3&&#AVV(((X0$/!-0#$&X-$"V52!1+V()*+K.+0X3&". '''',%+e'C'!/#2&E%+e ''''1)&)^0!g+'C'JNNNN '''',/)"+'C'Z&+0&X()*Z '''',%)&+'C'QQNW_XN '''')"#'C'aJNNNXN ''''/-3)//+.0'C'Q ''''0)"#(!1&3'C'W '''',%)"+%)&+'C'!/&9,%)&+@ ''''/,%)"+0'C'1)&)^0!g+ ''''-$"#&4#+'C'Z:M:hZ ''''-$"#/)"+'C'Z/$&'-$"#%+00+1Z ''''1)&)'C'R")&3X0!/9W'i'")&3X#!'i',%+e'i'9L'V',%)&+@@ '''''''''''',$%'L'!/'%)/5+91)&)^0!g+@T ''''()*^K.+'C'()*+X$#+/9,/)"+;'[([@ ''''()*^K.+X0+&#)%)"09 ''''''''9/-3)//+.0;'0)"#(!1&3;',%)"+%)&+;'/,%)"+0;'-$"#&4#+;'-$"#/)"+@@ '''',$%'*'!/'1)&)A ''''''''()*^K.+X(%!&+,%)"+090&%2-&X#)-H9[3[;'!/&9*'i')"#'V'W@@@ ''''()*^K.+X-.$0+9@ 1+,'+L)"#.+EE`9@A ''''1)&)^0!g+'C'JNNNN '''',/)"+'C'Z&+0&X()*Z ''''Y,%)&+'C'QQNW_XN ''''(0&)*^K.+'C'()*+X$#+/9,/)"+;'[%[@ '''',%)&+'C'()*^K.+X5+&,%)"+%)&+9@ ''''1)&)'C'()*^K.+X%+)1,%)"+091)&)^0!g+@ ''''()*^K.+X-.$0+9@ ''''1)&)'C'0&%2-&X2/#)-H9[j/k3[X,$%")&9/C1)&)^0!g+@;'1)&)@ ''''1)&)'C'/#X)%%)491)&)@ ''''Y,%)&+'C'()*+ ''''('C'/#X,,&X,,&91)&)@ '''',%+e0'C'/#X,,&X,,&,%+e9.+/9(@@ ''''#%!/&9,%+e0X"!/9@;',%+e0X")L9@@ ''''Y'9PNX_;'NXJlllf_@ ''''Y'E!/1'&3+'#+)H'!/'&3+'-$+m-!+/&0 ''''!1L'C'/#X)%5")L9/#X)609(@@ '''',%+e'C',%+e0R!1LT '''',%+e^!/^3+%&g'C')609,%+e'i',%)&+@ ''''#%!/&9,%+e^!/^3+%&g@ ''''Y'J8lXIlf_ 1+,']+)HE%+e9()*+B)&);'()*+])%)"+&+%0@A ''''1)&)'C'/#X)%%)49()*+B)&)@ ''''Y,%)&+'C'()*+ ''''('C'/#X,,&X,,&91)&)@ '''',%+e0'C'/#X,,&X,,&,%+e9.+/9(@@ ''''#%!/&9,%+e0X"!/9@;',%+e0X")L9@@ ''''!1L'C'/#X)%5")L9/#X)609(@@ '''',%+e'C',%+e0R!1LT '''',%+e^!/^3+%&g'C')609,%+e'i'()*+])%)"+&+%0X,%)"+%)&+@ ''''#%!/&9,%+e^!/^3+%&g@ ''''%+&2%/',%+e^!/^3+%&g 1+,'-.$-Hd)H+%9.+/5&3@A ''''-.$-HE!.+'C'RT '''',$%'L'!/'%)/5+9.+/5&3@A ''''''''!,'L'n'QNN'CC'NA ''''''''''''0)"#.+'C'JNNNN ''''''''+.0+A ''''''''''''0)"#.+'C'N ''''''''-.$-HE!.+X)##+/190)"#.+@ ''''%+&2%/'-.$-HE!.+ 1+,'")H+])%)"+&+%09@A ''''#`2#.+'C'/)"+1&2#.+9[^()*+^#)%)"0[;'R[/-3)//+.0[;'[0)"#(!1&3[;'[,%)"+%)&+[;'[/,%)"+0[;'[-$"#&4#+[;'[-$"#/)"+[T@ ''''#)%)"+&+%0'C'#`2#.+9/-3)//+.0CQ;'0)"#(!1&3CW;',%)"+%)&+CJJQNN;'/,%)"+0CJJQNNN;'-$"#&4#+C[:M:h[;'-$"#/)"+C[/$&'-$"#%+00+1[@ ''''%+&2%/'#)%)"+&+%0 1+,'-.+)/B)&)9!/#2&D)*+@A ''''-.+)/+1D)*+'C'RT '''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''!,'!/#2&D)*+RLT'U'N')/1'!/#2&D)*+RLPQT'CC'NA ''''''''''''0)"#.+'C'JNNNN ''''''''+.0+A ''''''''''''0)"#.+'C'N ''''''''-.+)/+1D)*+X)##+/190)"#.+@ ''''%+&2%/'-.+)/+1D)*+ 1+,'-.$-HG$2/&9!/#2&D)*+@A ''''12%)&!$/'C'RT ''''4'C'N '''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''!,'L'O'.+/9!/#2&D)*+@'P'QA ''''''''''''!,'!/#2&D)*+RLSQT'CC'JNNNNA ''''''''''''''''4'C'4'S'Q ''''''''''''''''12%)&!$/X)##+/194@ ''''''''''''''''4'C'N ''''''''''''+.0+A ''''''''''''''''4'C'4'S'Q ''''''''+.!,'L'CC'.+/9!/#2&D)*+@'P'QA ''''''''''''4'C'4'S'Q ''''''''''''12%)&!$/X)##+/194@ ''''%+&2%/'12%)&!$/ 1+,'-.$-HE!L9!/#2&D)*+;'-.$-HG$2/&@A ''''0&)%&'C'N ''''Y,$%'L'!/'%)/5+9.+/9-.$-HG$2/&@@A ''''KL+1D)*+'C'RT '''',$%'L'!/'%)/5+9.+/9-.$-HG$2/&@@A ''''''''+/1'C'0&)%&'S'-.$-HG$2/&RLT'P'Q ''''''''Y#%!/&90&%90&)%&@'S'['['S'0&%9+/1@@ ''''''''LB!*'C'QNN'V'-.$-HG$2/&RLT ''''''''4G).-'C'!/#2&D)*+R0&)%&A+/1'S'QT ''''''''LG).-'C'.!0&9%)/5+9N;'-.$-HG$2/&RLT@@ '''''''',$%'H'!/'%)/5+9.+/9LG).-@@A ''''''''''''LG).-RHT'C'LG).-RHT'i'LB!* ''''''''0#.!/+'C'!/&+%#$.)&+X!/&+%#Q19LG).-;'4G).-;'H!/1C[e2)1%)&!-[@ '''''''',$%'L'!/'%)/5+9N;'QNN@A ''''''''''''!,'L'CC'NA ''''''''''''''''0)"#.+G).-'C'!/#2&D)*+R0&)%&T ''''''''''''+.!,'L'CC'llA ''''''''''''''''0)"#.+G).-'C'!/#2&D)*+R+/1T ''''''''''''+.0+A ''''''''''''''''0)"#.+G).-'C'!/&90#.!/+9L@@ ''''''''''''KL+1D)*+X)##+/190)"#.+G).-@ ''''''''0&)%&'C'+/1'S'Q ''''%+&2%/'KL+1D)*+ Y'-%+)&+0'0&)&!-'*)%!0#++1')/1'KL+0'!&X 1+,'&+0&7##9*)%!0#++17"$2/&@A '''')21!$D)*+;'#)%)"+&+%0'C'$#+/D)*+E!.+9@ ''''-.$-HD)*+'C'-.$-Hd)H+%9.+/9)21!$D)*+@@ '''')21!$D)*+'C'*)%!0#++19)21!$D)*+;'#)%)"+&+%0;'*)%!0#++17"$2/&@ ''''-.$-HD)*+'C'*)%!0#++19-.$-HD)*+;'#)%)"+&+%0;'*)%!0#++17"$2/&@ ''''-.$-HD)*+'C'-.+)/B)&)9-.$-HD)*+@ ''''-.$-H'C'-.$-HG$2/&9-.$-HD)*+@ ''''KL+1D)*+'C'-.$-HE!L9)21!$D)*+;'-.$-H@ ''''%+&2%/'KL+1D)*+;'#)%)"+&+%0 Y-%+)&+0'*)%!)6.+'*)%!0#++1')/1'KL+0'!&X 1+,'&+0&7##W9*!6%)&$E%+e2+/-4;'*!6%)&$o/&+/0!&4@A '''')21!$D)*+;'#)%)"+&+%0'C'$#+/D)*+E!.+9@ ''''-.$-HD)*+'C'-.$-Hd)H+%9.+/9)21!$D)*+@@ '''')21!$D)*+'C'*!6%)&$721!$9)21!$D)*+;'#)%)"+&+%0;'*!6%)&$E%+e2+/-4;'*!6%)&$o/&+/0!&4@ ''''-.$-HD)*+'C'*!6%)&$721!$9-.$-HD)*+;'#)%)"+&+%0;'*!6%)&$E%+e2+/-4;'*!6%)&$o/&+/0!&4@ ''''-.$-HD)*+'C'-.+)/B)&)9-.$-HD)*+@ ''''-.$-H'C'-.$-HG$2/&9-.$-HD)*+@ ''''Y#%!/&9-.$-HRNAQNNNNT@ ''''KL+1D)*+'C'-.$-HE!L9)21!$D)*+;'-.$-H@ ''''%+&2%/'KL+1D)*+;'#)%)"+&+%0;')21!$D)*+ ''''Y%+&2%/'4G).-;'LG).- ''''''''YLB!*'C'QNN'V'-.$-HG$2/&RLT ''''''''Y+/1'C'-2%%+/&<)"#.+'S'-.$-HG$2/&RLT ''''''''Y4G).-'C'!/#2&D)*+R-2%%+/&<)"#.+A+/1T ''''''''YLG).-'C'.!0&9%)/5+9N;'-.$-HG$2/&RLT@@ ''''''''Y,$%'H'!/'%)/5+9.+/9LG).-@@A ''''''''Y''''LG).-RHT'C'LG).-RHT'i'LB!* ''''''''Y%+&2%/'LG).-;'4G).- 1+,'.$)1D)*+E!.+9K.+/)"+@A ''''%)(E!.+'C'()*+X$#+/9K.+/)"+;'"$1+C[%6[@ ''''%)(B2"#'C'%)(E!.+X%+)1,%)"+09%)(E!.+X5+&/,%)"+09@@ ''''#)%)"+&+%0'C'%)(E!.+X5+&#)%)"09@ ''''!/&+5+%D)*+'C'RT '''',$%'L'!/'%)/5+9%)(E!.+X5+&/,%)"+09@iW@A ''''''''!,'L'n'W'CC'NA ''''''''''''4'C'L'S'W ''''''''''''!/&+5+%D)*+X)##+/19!/&X,%$"^64&+09%)(B2"#RLA4T;'64&+$%1+%C[.!&&.+[;'0!5/+1C`%2+@@ ''''%+&2%/'!/&+5+%D)*+;'#)%)"+&+%0 1+,'$#+/D)*+E!.+9@A ''''!/#2&D)*+E!.+'C'[[ ''''!/#2&D)*+E!.+'C'!/#2&9Zp/D3!-3'()*+'K.+'1$'4$2'()/&'&$'#%$-+00q'9+L)"#.+A'[()*+X()*[@A'Z@ ''''&%4A ''''''''%)(E!.+'C'()*+X$#+/9!/#2&D)*+E!.+;'"$1+C[%6[@ ''''+L-+#&'E!.+:$&E$2/1h%%$%A ''''''''#)00 ''''%)(B2"#'C'%)(E!.+X%+)1,%)"+09%)(E!.+X5+&/,%)"+09@@ ''''#)%)"+&+%0'C'%)(E!.+X5+&#)%)"09@ ''''!/&+5+%D)*+'C'RT '''',$%'L'!/'%)/5+9%)(E!.+X5+&/,%)"+09@iW@A ''''''''!,'L'n'W'CC'NA ''''''''''''4'C'L'S'W ''''''''''''!/&+5+%D)*+X)##+/19!/&X,%$"^64&+09%)(B2"#RLA4T;'64&+$%1+%C[.!&&.+[;'0!5/+1C`%2+@@ ''''%+&2%/'!/&+5+%D)*+;'#)%)"+&+%0 1+,'+L#$%&721!$9!/#2&D)*+;'#)%)"+&+%0;'$2&#2&D)*+@A ''''64&+7%%)4D)*+'C'64&+)%%)49@ '''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''-2%%+/&o/&'C'!/#2&D)*+RLT ''''''''!,'-2%%+/&o/&'U'8WfafA ''''''''''''-2%%+/&o/&'C'8Wfaf ''''''''+.!,'-2%%+/&o/&'O'P8WfaIA ''''''''''''-2%%+/&o/&'C'P8WfaI ''''''''-2%%+/&r4&+'C'0&%2-&X#)-H9[O3[;'-2%%+/&o/&@ ''''''''64&+7%%)4D)*+X)##+/19-2%%+/&r4&+RNT@ ''''''''64&+7%%)4D)*+X)##+/19-2%%+/&r4&+RQT@ ''''(E!.+'C'()*+X$#+/9$2&#2&D)*+;'"$1+C[(6[@ ''''(E!.+X0+&#)%)"09#)%)"+&+%0@ ''''(E!.+X(%!&+,%)"+0%)(964&+7%%)4D)*+@ ''''(E!.+X-.$0+9@ 1+,'+L#$%&721!$E=9!/#2&D)*+;',%)"+=)&+;'$2&#2&D)*+@A ''''64&+7%%)4D)*+'C'64&+)%%)49@ '''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''-2%%+/&o/&'C'!/#2&D)*+RLT ''''''''!,'-2%%+/&o/&'U'8WfafA ''''''''''''-2%%+/&o/&'C'8Wfaf ''''''''+.!,'-2%%+/&o/&'O'P8WfaIA ''''''''''''-2%%+/&o/&'C'P8WfaI ''''''''-2%%+/&r4&+'C'0&%2-&X#)-H9[O3[;'-2%%+/&o/&@ ''''''''64&+7%%)4D)*+X)##+/19-2%%+/&r4&+RNT@ ''''''''64&+7%%)4D)*+X)##+/19-2%%+/&r4&+RQT@ ''''(E!.+'C'()*+X$#+/9$2&#2&D)*+;'"$1+C[(6[@ ''''(E!.+X0+&/-3)//+.09Q@ ''''(E!.+X0+&0)"#(!1&39W@ ''''(E!.+X0+&,%)"+%)&+9,%)"+=)&+@ ''''(E!.+X(%!&+,%)"+0%)(964&+7%%)4D)*+@ ''''(E!.+X-.$0+9@ 1+,'&+-3b)%!0#++19!/#2&D)*+;'#)%)"+&+%0;'%+0)"#.+=)&+@A ''''!/#2&<)"#.+=)&+'C'#)%)"+&+%0RWT ''''%+0)"#.+=+.)&!*+=)&+'C'9QV9\$)&9%+0)"#.+=)&+@VQNN@@'i'!/#2&<)"#.+=)&+ ''''!/#2&<)"#.+G$2/&'C'.+/9!/#2&D)*+@ ''''%+0)"#.+=)&!$7'C'%+0)"#.+=)&+'V'!/#2&<)"#.+=)&+ ''''%+0)"#.+=)&!$r'C'!/#2&<)"#.+=)&+'V'%+0)"#.+=)&+ ''''%+0)"#.+<)"#.+G$2/&'C'!/&9!/#2&<)"#.+G$2/&'i'%+0)"#.+=)&!$7@ ''''/+(<)"#.+'C'N ''''#%$-+00+1D)*+'C'RT '''',$%'L'!/'%)/5+9%+0)"#.+<)"#.+G$2/&@A ''''''''!,'L'i'%+0)"#.+=)&!$r'O'!/#2&<)"#.+G$2/&'P'QA ''''''''''''L]$!/&+%'C'L'i'%+0)"#.+=)&!$r ''''''''''''0)"#.+]$!/&+%'C'!/&9L]$!/&+%@ ''''''''''''.!/+)%o/&]$!/&'C'9L]$!/&+%@'n'Q ''''''''''''-2%%+/&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%T ''''''''''''/+L&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%'S'QT ''''''''''''/+L&<)"#.+B!,,'C'/+L&<)"#.+'P'-2%%+/&<)"#.+ '''''''''''')"$2/&r+&(++/'C'/+L&<)"#.+B!,,'i'.!/+)%o/&]$!/& ''''''''''''-2%%+/&o/&+%#$.)&!$/'C'-2%%+/&<)"#.+'S')"$2/&r+&(++/ ''''''''''''/+(<)"#.+'C'!/&9%$2/19-2%%+/&o/&+%#$.)&!$/@@ ''''''''#%$-+00+1D)*+X)##+/19/+(<)"#.+@ ''''%+&2%/'#%$-+00+1D)*+ 1+,'*)%!0#++19!/#2&D)*+;'#)%)"+&+%0;'%+0)"#.+=)&+@A ''''!/#2&<)"#.+=)&+'C'#)%)"+&+%0RWT ''''%+0)"#.+=+.)&!*+=)&+'C'9QV9\$)&9%+0)"#.+=)&+@VQNN@@'i'!/#2&<)"#.+=)&+ ''''!/#2&<)"#.+G$2/&'C'.+/9!/#2&D)*+@ ''''%+0)"#.+=)&!$7'C'%+0)"#.+=+.)&!*+=)&+'V'!/#2&<)"#.+=)&+ ''''%+0)"#.+=)&!$r'C'!/#2&<)"#.+=)&+'V'%+0)"#.+=+.)&!*+=)&+ ''''%+0)"#.+<)"#.+G$2/&'C'!/&9!/#2&<)"#.+G$2/&'i'%+0)"#.+=)&!$7@ ''''/+(<)"#.+'C'N ''''#%$-+00+1D)*+'C'RT '''',$%'L'!/'%)/5+9%+0)"#.+<)"#.+G$2/&@A ''''''''!,'L'O'%+0)"#.+<)"#.+G$2/&'P'QA ''''''''''''L]$!/&+%'C'L'i'%+0)"#.+=)&!$r ''''''''''''0)"#.+]$!/&+%'C'!/&9L]$!/&+%@ ''''''''''''.!/+)%o/&]$!/&'C'9L]$!/&+%@'n'Q ''''''''''''-2%%+/&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%T ''''''''''''/+L&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%'S'QT ''''''''''''/+L&<)"#.+B!,,'C'/+L&<)"#.+'P'-2%%+/&<)"#.+ '''''''''''')"$2/&r+&(++/'C'/+L&<)"#.+B!,,'i'.!/+)%o/&]$!/& ''''''''''''-2%%+/&o/&+%#$.)&!$/'C'-2%%+/&<)"#.+'S')"$2/&r+&(++/ ''''''''''''/+(<)"#.+'C'!/&9%$2/19-2%%+/&o/&+%#$.)&!$/@@ ''''''''+.!,'L'CC'%+0)"#.+<)"#.+G$2/&'P'QA ''''''''''''/+(<)"#.+'C'!/#2&D)*+RPQT ''''''''#%$-+00+1D)*+X)##+/19/+(<)"#.+@ ''''%+&2%/'#%$-+00+1D)*+ 1+,'*!6%)&$721!$9!/#2&D)*+;'#)%)"+&+%0;'*!6%)&$E%+e2+/-4;'*!6%)&$o/&+/0!&4@A ''''*!6%)&$E%+e2+/-4'C'\$)&9*!6%)&$E%+e2+/-4@ ''''*!6%)&$o/&+/0!&4'C'\$)&9*!6%)&$o/&+/0!&4@ ''''!/#2&<)"#.+=)&+'C'#)%)"+&+%0RWT ''''!/#2&<)"#.+G$2/&'C'.+/9!/#2&D)*+@ '''',%+e=)1'C'")&3X&)2'V'!/#2&<)"#.+=)&+ ''''#3)0+o/-%'C',%+e=)1'i'*!6%)&$E%+e2+/-4 ''''#3)0+'C'N ''''#%$-+00+1D)*+'C'RT '''',$%'L'!/'%)/5+9!/#2&<)"#.+G$2/&@A ''''''''-2%%+/&o/1+L'C'L'S'")&3X0!/9#3)0+@'i'*!6%)&$o/&+/0!&4 ''''''''#3)0+'C'#3)0+'S'#3)0+o/-% ''''''''!,'#3)0+'UC'")&3X&)2A ''''''''''''#3)0+'C'#3)0+'P'")&3X&)2 ''''''''!,'-2%%+/&o/1+L'OC'!/#2&<)"#.+G$2/&'P'QA ''''''''''''L]$!/&+%'C'-2%%+/&o/1+L ''''''''''''0)"#.+]$!/&+%'C'!/&9L]$!/&+%@ ''''''''''''.!/+)%o/&]$!/&'C'9L]$!/&+%@'n'Q ''''''''''''-2%%+/&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%T ''''''''''''/+L&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%'S'QT ''''''''''''/+L&<)"#.+B!,,'C'/+L&<)"#.+'P'-2%%+/&<)"#.+ '''''''''''')"$2/&r+&(++/'C'/+L&<)"#.+B!,,'i'.!/+)%o/&]$!/& ''''''''''''-2%%+/&o/&+%#$.)&!$/'C'-2%%+/&<)"#.+'S')"$2/&r+&(++/ ''''''''''''/+(<)"#.+'C'!/&9%$2/19-2%%+/&o/&+%#$.)&!$/@@ ''''''''''''#%$-+00+1D)*+X)##+/19/+(<)"#.+@ ''''%+&2%/'#%$-+00+1D)*+ 1+,'*)%!b!6%)&$9!/#2&D)*+;'#)%)"+&+%0;'*!6%)&$E%+e2+/-4;'*!6%)&$o/&+/0!&4;'"$1E%+e2+/-4;'"$1o/&+/0!&4@A ''''!/#2&<)"#.+=)&+'C'#)%)"+&+%0RWT ''''!/#2&<)"#.+G$2/&'C'.+/9!/#2&D)*+@ ''''*!6%)&$E%+e2+/-4'C'\$)&9*!6%)&$E%+e2+/-4@ ''''*!6%)&$o/&+/0!&4'C'\$)&9*!6%)&$o/&+/0!&4@ ''''"$1E%+e2+/-4'C'\$)&9"$1E%+e2+/-4@ ''''"$1o/&+/0!&4'C'\$)&9"$1o/&+/0!&4@ '''',%+e=)1'C'")&3X&)2'V'!/#2&<)"#.+=)&+ ''''#3)0+o/-%'C',%+e=)1'i'*!6%)&$E%+e2+/-4 ''''"$1]3)0+o/-%'C',%+e=)1'i'"$1E%+e2+/-4 ''''#3)0+'C'N ''''"$1]3)0+'C'N ''''#%$-+00+1D)*+'C'RT '''',$%'L'!/'%)/5+9!/#2&<)"#.+G$2/&@A ''''''''-2%%+/&o/1+L'C'L'S'")&3X0!/9#3)0+@'i'*!6%)&$o/&+/0!&4 ''''''''"$1b).2+'C'"$1o/&+/0!&4'i'")&3X0!/9"$1]3)0+@ ''''''''#3)0+'C'#3)0+'S'#3)0+o/-%'S'"$1b).2+ ''''''''"$1]3)0+'C'"$1]3)0+'S'"$1]3)0+o/-% ''''''''Y#%!/&9-2%%+/&o/1+L@ ''''''''!,'#3)0+'UC'")&3X&)2A ''''''''''''#3)0+'C'#3)0+'P'")&3X&)2 ''''''''!,'-2%%+/&o/1+L'OC'!/#2&<)"#.+G$2/&'P'QA ''''''''''''L]$!/&+%'C'-2%%+/&o/1+L ''''''''''''0)"#.+]$!/&+%'C'!/&9L]$!/&+%@ ''''''''''''.!/+)%o/&]$!/&'C'9L]$!/&+%@'n'Q ''''''''''''-2%%+/&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%T ''''''''''''/+L&<)"#.+'C'!/#2&D)*+R0)"#.+]$!/&+%'S'QT ''''''''''''/+L&<)"#.+B!,,'C'/+L&<)"#.+'P'-2%%+/&<)"#.+ '''''''''''')"$2/&r+&(++/'C'/+L&<)"#.+B!,,'i'.!/+)%o/&]$!/& ''''''''''''-2%%+/&o/&+%#$.)&!$/'C'-2%%+/&<)"#.+'S')"$2/&r+&(++/ ''''''''''''/+(<)"#.+'C'!/&9%$2/19-2%%+/&o/&+%#$.)&!$/@@ ''''''''''''#%$-+00+1D)*+X)##+/19/+(<)"#.+@ ''''%+&2%/'#%$-+00+1D)*+ 1+,'%+*+%0+9!/#2&D)*+@A ''''#%$-+00+1D)*+'C'RT '''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''/+(<)"#.+'C'!/#2&D)*+RP9LSQ@T ''''''''#%$-+00+1D)*+X)##+/19/+(<)"#.+@ ''''%+&2%/'#%$-+00+1D)*+ 1+,'0!"#.+E!.&+%9!/#2&D)*+;'K.&+%`4#+@A ''''#%$-+00+1D)*+'C'RT ''''!,'K.&+%`4#+'CC'[/$&-3[A ''''''''K.&+%<)"#.+'C'N '''''''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''''''!,'L'O'.+/9!/#2&D)*+@'P'WA ''''''''''''''''K.&+%<)"#.+'C'!/&99!/#2&D)*+RLT'S'!/#2&D)*+RLSWT@'V'W@ ''''''''''''#%$-+00+1D)*+X)##+/19K.&+%<)"#.+@ ''''''''YD>7`'`M'BM'EM='?7<`'<7d]?hq ''''+.!,'K.&+%`4#+'CC'[.$(#)00[A ''''''''K.&+%<)"#.+'C'N '''''''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''''''!,'L'O'.+/9!/#2&D)*+@'P'QA ''''''''''''''''K.&+%<)"#.+'C'!/&99!/#2&D)*+RLT'S'!/#2&D)*+RLSQT@'V'W@ ''''''''''''#%$-+00+1D)*+X)##+/19K.&+%<)"#.+@ ''''+.!,'K.&+%`4#+'CC'[3!53#)00[A ''''''''K.&+%<)"#.++'C'N '''''''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''''''!,'L'O'.+/9!/#2&D)*+@'P'QA ''''''''''''''''K.&+%<)"#.+'C'!/&99!/#2&D)*+RLT'P'!/#2&D)*+RLSQT@'V'W@ ''''''''''''#%$-+00+1D)*+X)##+/19K.&+%<)"#.+@ ''''+.!,'K.&+%`4#+'CC'[6)/1#)00[A ''''''''K.&+%<)"#.+'C'N '''''''',$%'L'!/'%)/5+9.+/9!/#2&D)*+@@A ''''''''''''!,'L'O'.+/9!/#2&D)*+@'P'WA ''''''''''''''''K.&+%<)"#.+'C'!/&99!/#2&D)*+RLT'P'!/&+5+%D)*+RLSWT@'V'W@ ''''''''''''#%$-+00+1D)*+X)##+/19K.&+%<)"#.+@ ''''%+&2%/'#%$-+00+1D)*+ Y0!"#.+E!.&+%9[s20&7/$&3+%B)4QNX()*[;'[6)/1#)00[;'[0!"#.+E!.&+%r)/1#)00NQX()*[@ 1+,'62&&+%D$%&37##8IJ9!/&+5+%D)*+@A ''''K.&+%E%+e'C'R8_NNN'V'QlWNNNT ''''!/&+5+%D)*+'C'62&&+%($%&3E!.&+%9!/&+5+%D)*+;'a;'[3!53#)00[;'K.&+%E%+e@ ''''%+&2%/'!/&+5+%D)*+ 1+,'62&&+%($%&3E!.&+%9!/#2&D)*+;'K.&+%M%1+%;'K.&+%`4#+;'-2&$,,E%+e@A ''''K.&+%M%1+%'C'!/&9K.&+%M%1+%@ ''''6;')'C'0!5/).X62&&+%9K.&+%M%1+%;'-2&$,,E%+e;'K.&+%`4#+@ ''''#%$-+00+1D)*+'C'0!5/).XK.&K.&96;');'!/#2&D)*+@X&$.!0&9@ '''',$%'L'!/'%)/5+9.+/9#%$-+00+1D)*+@@A ''''''''#%$-+00+1D)*+RLT'C'!/&9%$2/19#%$-+00+1D)*+RLT@@ ''''%+&2%/'#%$-+00+1D)*+ ''''Y62&&+%($%&3E!.&+%9[s20&7/$&3+%B)4QNX()*[;'a;'[6)/1#)00[;'RQVWN;'QVQNT;'[62&&+%($%&3r)/1#)00NQX()*[@ 1+,')21!$<2"'902"o/#2&Q;'02"?+*+.Q;'02"o/#2&W;'02"?+*+.W@A 02"D)*+'C'RT -2%%+/&<)"#.+'C'N ,$%'L'!/'%)/5+9.+/902"o/#2&Q@@A !,'L'O'.+/902"o/#2&W@A -2%%+/&<)"#.+'C'!/&9%$2/19902"o/#2&QRLT'i'02"?+*+.Q@'S'902"o/#2&WRLT'i'02"?+*+.W@@@ +.!,'L'UC'.+/902"o/#2&W@A -2%%+/&<)"#.+'C'!/&9%$2/1902"o/#2&QRLT'i'02"?+*+.Q@@ 02"D)*+X)##+/19-2%%+/&<)"#.+@ %+&2%/'02"D)*+ 1+,')21!$d!L9!/#2&7;'.+*+.7;'!/#2&r;'.+*+.r@A !,'.+/9!/#2&7@'UC'.+/9!/#2&r@A #%$-+00+1D)*+'C')21!$<2"9!/#2&7;'.+*+.7;'!/#2&r;'.+*+.r@ +.0+A #%$-+00+1D)*+'C')21!$<2"9!/#2&r;'.+*+.r;'!/#2&7;'.+*+.7@ %+&2%/'#%$-+00+1D)*+ Y)21!$d!L9[!/#2&7X()*[;'X_;'[!/#2&rX()*[;'X_;'[)21!$d!LNQX()*[@ [[[ !,'^^/)"+^^'CC'Z^^")!/^^ZA ''''#%!/&9[p/iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiip/i'D+.-$"+'&$'&3+'721!$']%$-+00$%'ip/iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiip/p/G2%%+/&'*+%0!$/'($%H0'$/.4'(!&3'Qa6!&'"$/$'X()*'K.+0X[@ ''''!/&+5+%D)*+;'#)%)"+&+%0'C'$#+/D)*+E!.+9@ ''''#%$"#&'C'[[ ''''(3!.+'#%$"#&'tC'[N[A ''''''''#%$"#&'C'!/#2&9Zp/p&QA'b)%!0#++1p/p&WA'b!6%)&$p/p&8A'b)%!b!6%)&$p/p&JA'<!"#.+'E!.&+%0p/p&_A'r2&&+%($%&3'E!.&+%0p/p&aA'd!Lp/p&fA'=+*+%0+p/p&IA'hL#$%&'&$'K.+p/p&NA'hco`p/p/<+.+-&')'#%$-+00A'Z@ ''''''''!,'#%$"#&'CC'[I[A ''''''''''''$2&#2&D)*+'C'[[ ''''''''''''$2&#2&D)*+'C'!/#2&9Z:)"+'&3+'$2&#2&'K.+A'Z@ ''''''''''''+L#$%&721!$9!/&+5+%D)*+;'#)%)"+&+%0;'$2&#2&D)*+@ ''''''''!,'#%$"#&'CC'[Q[A ''''''''''''%+0)"#.+=)&+'C'[[ ''''''''''''%+0)"#.+=)&+'C'!/#2&9Z<#+-!,4'<#++1'n'9+L)"#.+A'[QNN['!0'$%!5!/).'%)&+@A'Z@ ''''''''''''!/&+5+%D)*+'C'*)%!0#++19!/&+5+%D)*+;'#)%)"+&+%0;'%+0)"#.+=)&+@ ''''''''!,'#%$"#&'CC'[W[A ''''''''''''*!6%)&$=)&+'C'[[ ''''''''''''*!6%)&$=)&+'C'!/#2&9Zh/&+%'b!6%)&$'E%+e2+/-4'9+L)"#.+A'[NXW[',$%')/'?EM@A'Z@ ''''''''''''*!6%)&$7"$2/&'C'[[ ''''''''''''*!6%)&$7"$2/&'C'!/#2&9Zh/&+%'b!6%)&$'7"$2/&'9+L)"#.+A'[QNNN[@A'Z@ ''''''''''''!/&+5+%D)*+'C'*!6%)&$721!$9!/&+5+%D)*+;'#)%)"+&+%0;'*!6%)&$=)&+;'*!6%)&$7"$2/&@ ''''''''!,'#%$"#&'CC'[8[A ''''''''''''#%!/&9[p/ip&b)%!+0'&3+'%)&+'$,'*!6%)&$p&i[@ ''''''''''''*!6%)&$=)&+'C'[[ ''''''''''''*!6%)&$=)&+'C'!/#2&9Zp/h/&+%'b!6%)&$'E%+e2+/-4'9+L)"#.+A'[NXQ[@A'Z@ ''''''''''''*!6%)&$7"$2/&'C'[[ ''''''''''''*!6%)&$7"$2/&'C'!/#2&9Zh/&+%'b!6%)&$'7"$2/&'9+L)"#.+A'[8NN[@A'Z@ ''''''''''''"$1E%+e2+/-4'C'[[ ''''''''''''"$1E%+e2+/-4'C'!/#2&9Zh/&+%'d$12.)&!$/'E%+e2+/-4'9+L)"#.+A'[XNQ[@A'Z@ ''''''''''''"$1o/&+/0!&4'C'[[ ''''''''''''"$1o/&+/0!&4'C'!/#2&9Zh/&+%'d$12.)&!$/'o/&+/0!&4'9+L)"#.+A'[XNNQ[@A'Z@ ''''''''''''!/&+5+%D)*+'C'*)%!b!6%)&$9!/&+5+%D)*+;'#)%)"+&+%0;'*!6%)&$=)&+;'*!6%)&$7"$2/&;'"$1E%+e2+/-4;'"$1o/&+/0!&4@ ''''''''!,'#%$"#&'CC'[J[A ''''''''''''#%!/&9[p/`3+0+'K.&+%0')%+'6)0!-'-$/*$.2&!$/',2/-&!$/0'&3)&p/1$'/$&')..$(',$%'-3)/5+0'!/'-2&$,,',%+e2+/-4'$%'0.$#+X[@ ''''''''''''K.&+%]%$"#&'C'[[ ''''''''''''K.&+%]%$"#&'C'!/#2&9Zp/p&QA'?$(#)00p/p&WA'>!53#)00p/p&8A'r)/1#)00p/p&JA':$&-3p/p/<+.+-&')'K.&+%'&4#+A'Z@ ''''''''''''!,'K.&+%]%$"#&'CC'[Q[A ''''''''''''''''!/&+5+%D)*+'C'0!"#.+E!.&+%9!/&+5+%D)*+;'[.$(#)00[@ ''''''''''''+.!,'K.&+%]%$"#&'CC'[W[A ''''''''''''''''!/&+5+%D)*+'C'0!"#.+E!.&+%9!/&+5+%D)*+;'[3!53#)00[@ ''''''''''''+.!,'K.&+%]%$"#&'CC'[8[A ''''''''''''''''!/&+5+%D)*+'C'0!"#.+E!.&+%9!/&+5+%D)*+;'[6)/1#)00[@ ''''''''''''+.!,'K.&+%]%$"#&'CC'[J[A ''''''''''''''''!/&+5+%D)*+'C'0!"#.+E!.&+%9!/&+5+%D)*+;'[/$&-3[@ ''''''''!,'#%$"#&'CC'[_[A ''''''''''''K.&+%`4#+]%$"#&'C'[[ ''''''''''''K.&+%`4#+]%$"#&'C'!/#2&9Zp/p&QA'?$(#)00p/p&WA'>!53#)00p/p&8A'r)/1#)00p/p&JA':$&-3p/p/<+.+-&')'K.&+%'&4#+A'Z@ ''''''''''''!,'K.&+%`4#+]%$"#&'CC'[Q[A ''''''''''''''''K.&+%`4#+'C'[.$(#)00[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'[[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'!/#2&9Zp/h/&+%')'K.&+%'$%1+%'QPaA'Z@ ''''''''''''''''K.&+%E%+e7]%$"#&'C'[[ ''''''''''''''''K.&+%E%+e7]%$"#&'C'!/#2&9Zp/h/&+%')'-2&$,,',%+e2+/-4'!/'>g'9+L)"#.+A'[JNN[@A'Z@ ''''''''''''''''K.&+%E%+e'C'R\$)&9K.&+%E%+e7]%$"#&@'V'9#)%)"+&+%0RWT'V'W@T ''''''''''''''''!/&+5+%D)*+'C'62&&+%($%&3E!.&+%9!/&+5+%D)*+;'K.&+%M%1+%]%$"#&;'K.&+%`4#+;'K.&+%E%+e@ ''''''''''''+.!,'K.&+%`4#+]%$"#&'CC'[W[A ''''''''''''''''K.&+%`4#+'C'[3!53#)00[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'[[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'!/#2&9Zp/h/&+%')'K.&+%'$%1+%'QPaA'Z@ ''''''''''''''''K.&+%E%+e7]%$"#&'C'[[ ''''''''''''''''K.&+%E%+e7]%$"#&'C'!/#2&9Zp/h/&+%')'-2&$,,',%+e2+/-4'!/'>g'9+L)"#.+A'[JNN[@A'Z@ ''''''''''''''''K.&+%E%+e'C'R\$)&9K.&+%E%+e7]%$"#&@'V'9#)%)"+&+%0RWT'V'W@T ''''''''''''''''!/&+5+%D)*+'C'62&&+%($%&3E!.&+%9!/&+5+%D)*+;'K.&+%M%1+%]%$"#&;'K.&+%`4#+;'K.&+%E%+e@ ''''''''''''+.!,'K.&+%`4#+]%$"#&'CC'[8[A ''''''''''''''''K.&+%`4#+'C'[6)/1#)00[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'[[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'!/#2&9Zp/h/&+%')'K.&+%'$%1+%'QPJA'Z@ ''''''''''''''''K.&+%E%+e7]%$"#&'C'[[ ''''''''''''''''K.&+%E%+e7]%$"#&'C'!/#2&9Zp/h/&+%')'.$(+%'-2&$,,',%+e2+/-4'!/'>g'9+L)"#.+A'[JNN[@A'Z@ ''''''''''''''''K.&+%E%+er]%$"#&'C'[[ ''''''''''''''''K.&+%E%+er]%$"#&'C'!/#2&9Zp/h/&+%')'3!53'-2&$,,',%+e2+/-4'!/'>g'9+L)"#.+A'[QNNN[@A'Z@ ''''''''''''''''K.&+%E%+e'C'R\$)&9K.&+%E%+e7]%$"#&@'V'9#)%)"+&+%0RWT'V'W@;'\$)&9K.&+%E%+er]%$"#&@'V'9#)%)"+&+%0RWT'V'W@T ''''''''''''''''!/&+5+%D)*+'C'62&&+%($%&3E!.&+%9!/&+5+%D)*+;'K.&+%M%1+%]%$"#&;'K.&+%`4#+;'K.&+%E%+e@ ''''''''''''+.!,'K.&+%`4#+]%$"#&'CC'[J[A ''''''''''''''''K.&+%`4#+'C'[6)/10&$#[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'[[ ''''''''''''''''K.&+%M%1+%]%$"#&'C'!/#2&9Zp/h/&+%')'K.&+%'$%1+%'QPJA'Z@ ''''''''''''''''K.&+%E%+e7]%$"#&'C'[[ ''''''''''''''''K.&+%E%+e7]%$"#&'C'!/#2&9Zp/h/&+%')'.$(+%'-2&$,,',%+e2+/-4'!/'>g'9+L)"#.+A'[JNN[@A'Z@ ''''''''''''''''K.&+%E%+er]%$"#&'C'[[ ''''''''''''''''K.&+%E%+er]%$"#&'C'!/#2&9Zp/h/&+%')'3!53'-2&$,,',%+e2+/-4'!/'>g'9+L)"#.+A'[QNNN[@A'Z@ ''''''''''''''''K.&+%E%+e'C'R\$)&9K.&+%E%+e7]%$"#&@'V'9#)%)"+&+%0RWT'V'W@;'\$)&9K.&+%E%+er]%$"#&@'V'9#)%)"+&+%0RWT'V'W@T ''''''''''''''''!/&+5+%D)*+'C'62&&+%($%&3E!.&+%9!/&+5+%D)*+;'K.&+%M%1+%]%$"#&;'K.&+%`4#+;'K.&+%E%+e@ ''''''''!,'#%$"#&'CC'[a[A ''''''''''''"!L?+*+.7]%$"#&'C'[[ ''''''''''''"!L?+*+.7]%$"#&'C'!/#2&9[p/h/&+%')'"!L'.+*+.',$%'.$)1+1'()*+'9NPQNN@A'[@ ''''''''''''"!L?+*+.7]%$"#&'C'\$)&9"!L?+*+.7]%$"#&@'V'QNN ''''''''''''#%!/&9Zp/<+.+-&')'D)*+'E!.+'&$'d!Lp/Z@ ''''''''''''!/&+5+%D)*+r;'#)%)"+&+%0r'C'$#+/D)*+E!.+9@ ''''''''''''"!L?+*+.r]%$"#&'C'[[ ''''''''''''"!L?+*+.r]%$"#&'C'!/#2&9[p/h/&+%')'"!L'.+*+.',$%'.$)1+1'()*+'9NPQNN@A'[@ ''''''''''''"!L?+*+.r]%$"#&'C'\$)&9"!L?+*+.r]%$"#&@'V'QNN ''''''''''''!/&+5+%D)*+'C')21!$d!L9!/&+5+%D)*+r;'"!L?+*+.r]%$"#&;'!/&+5+%D)*+;'"!L?+*+.7]%$"#&@ ''''''''!,'#%$"#&'CC'[f[A ''''''''''''!/&+5+%D)*+'C'%+*+%0+9!/&+5+%D)*+@ [[[ The Paper i 2020 Sound by Finlay Braithwaite A thesis exhibition presented to OCAD University in partial fulfillment of the requirements for the degree of Master of Design in Digital Futures Toronto Media Arts Centre, April 12 - 14 2019 Toronto, Ontario, Canada, April , 2019 © Finlay Braithwaite 2019 ii Author’s Declaration I hereby declare that I am the sole author of this thesis. This is a true copy of the thesis, including any required final revisions, as accepted by my examiners. I auth orize OCAD University to lend this thesis to other institutions or individuals for the purpose of scholarly research. I understand that my thesis may be made electronically available to the public. I further authorize OCAD University to reproduce this thes is by photocopying or by other means, in total or in part, at the request of other institutions or individuals for the purpose of scholarly research. Signature __________________________________________________ iii Abstract 2020 Sound is a positionally - aware microphone and DSP time - of - arrival alignment system. 2020 Sound is both a production positional tracking tool and post production alignment process. In product ion, an ultrasonic beacon emits a temporal positional reference that is captured by standard audio recording devices. This reference gives purchase to align multiple microphone perspectives of a source, correcting for their initial offset as well their mov ement throughout the recording. In capturing a sound source with multiple microphones, misaligned and drifting time - of - arrival of the source at each microphone greatly impacts the cohesion, focus, and impact of their summation in the mixing process. The co mmon boom and lavaliere microphone scenario implemented in film and television production suffers from this misalignment and, as a result, time - intensive and inaccurate manual editorial processes are employed to align microphones before their summation. Th is system could remedy a fundamental issue encountered in audio production with the ultimate aim of improving the clarity and quality of productions that make use 2020 Sound. ii Table of Contents Author’s Declaration ................................ ................................ ................................ ....................... ii Abstract ................................ ................................ ................................ ................................ .......... iii 1 Introduction ................................ ................................ ................................ ............................. 1 2 Properties of Sound ................................ ................................ ................................ ................. 4 2.1 Sound Fundamentals ................................ ................................ ................................ ........ 4 2.2 Freque ncy and Wavelength ................................ ................................ ............................. 5 2.3 Time of Arrival ................................ ................................ ................................ .................. 5 2.4 Reflections ................................ ................................ ................................ ........................ 6 2.5 Summing multiple perspectives ................................ ................................ ....................... 7 2.6 Comb filtering ................................ ................................ ................................ ................... 8 2.7 Spatial Audio ................................ ................................ ................................ .................... 9 3 Time of Arrival Narratives ................................ ................................ ................................ ...... 11 3.1 Production ................................ ................................ ................................ ...................... 11 3.1.1 Two Sources, one perspective ................................ ................................ ................ 13 3.2 Post Production ................................ ................................ ................................ .............. 14 3.2.1 Time of Arrival Alignment ................................ ................................ ....................... 15 3.3 Analogous Problems ................................ ................................ ................................ ....... 17 3.3.1 Issues in Music ................................ ................................ ................................ ........ 17 iii 3.3.2 Wow and Flutter in Analog Record ings ................................ ................................ .. 19 3.3.3 Audio System Latencies ................................ ................................ .......................... 21 4 Methodology ................................ ................................ ................................ ......................... 23 4.1 Narrative - Driven User - Centred Design ................................ ................................ .......... 24 4.2 Research throu gh Design ................................ ................................ ............................... 24 4.3 Iterative Prototyping ................................ ................................ ................................ ...... 25 4.4 Learning by Teaching ................................ ................................ ................................ ...... 26 5 Existing Designs ................................ ................................ ................................ ..................... 27 5.1 Celemony Capstan ................................ ................................ ................................ .......... 27 5.2 Dan Duga n Automixer ................................ ................................ ................................ .... 28 5.3 Syncro Arts VocAlign ................................ ................................ ................................ ...... 30 5.4 Sound Radix Auto Align Post ................................ ................................ .......................... 32 6 2020 Sound ................................ ................................ ................................ ............................ 34 6.1 Conceptual Framework ................................ ................................ ................................ .. 34 6.2 Production ................................ ................................ ................................ ...................... 35 6.3 Post Production ................................ ................................ ................................ .............. 37 6.4 User Narrative Design ................................ ................................ ................................ .... 39 6.4.1 Production ................................ ................................ ................................ ............... 39 6.4.2 Post Production ................................ ................................ ................................ ....... 40 iv 7 Prototyping ................................ ................................ ................................ ............................ 41 7.1 Ultrasonic Clocking ................................ ................................ ................................ ......... 41 7.2 DSP Alignment System ................................ ................................ ................................ ... 43 7.2.1 A Dummy Clock ................................ ................................ ................................ ....... 43 7.2.2 Clock Distortion ................................ ................................ ................................ ....... 44 7.2.3 Alignment Algorithm ................................ ................................ ............................... 44 7.2.4 Parallel Processing ................................ ................................ ................................ .. 45 7.2.5 Modulating Temporal Distortion ................................ ................................ ............ 46 7.2.6 Real Clocks ................................ ................................ ................................ .............. 47 7.2.7 Feature Detection ................................ ................................ ................................ ... 48 7.2.8 Clock with Audio ................................ ................................ ................................ ..... 50 8 Conclusions and Future Work ................................ ................................ ............................... 52 Bibliography ................................ ................................ ................................ ................................ .. 54 A ppendix ................................ ................................ ................................ ................................ ........ vi Methods ................................ ................................ ................................ ................................ ..... vi Clock signals ................................ ................................ ................................ ............................ vi Interpolation ................................ ................................ ................................ ............................ x Feature Extraction ................................ ................................ ................................ ................. xii v Figure 1 - Constructive and Destructive Interference of Two Summed Microphones . ................................ 1 Figure 2 - Amplitude, Phase, and Wav elength of Sound ................................ ................................ ............... 5 Figure 3 – Time - of - arrival of two sources with common signal ................................ ................................ ... 6 Figure 4 - Summation of sine wave in varying phases ................................ ................................ ................... 8 Figure 5 – Co mb - filtering resultant of two microphones at varying distances from source ........................ 9 Figure 6 - Dialogue Editorial Alignment ................................ ................................ ................................ ...... 16 Figure 7 - Pitch variation: uncorrected (left) and corrected (right) ................................ ........................... 20 Figure 8 - Dan Dugan Automixer for Waves Multirack ................................ ................................ ............... 29 Figure 9 - VocAlign PRO 4 ................................ ................................ ................................ ............................ 31 Figure 10 - Sound Radix Auto Align Post ................................ ................................ ................................ ...... 32 Figure 11 - Production Design Overview ................................ ................................ ................................ ..... 35 Figure 12 - Post Produc tion Design Overview ................................ ................................ ............................. 37 Figure 13 - User and Process Flow ................................ ................................ ................................ .............. 37 Figure 14 - User Interface ................................ ................................ ................................ ........................... 38 Figure 15 - Manchester biphase encoding ................................ ................................ ................................ . viii Figure 16 - Frequency modulated chirp sweeping up in frequency ................................ ............................ ix Figure 17 - Time of Arrival positional localization ................................ ................................ ........................ x Figure 18 - Zero Order Hold, Linear, Polynomial Interpolation ................................ ................................ .. xi 1 1 Introduction This research through design project explores possible solutions for the alignment of time - of - arrival for a sound source arriving at multiple microphones. Scaffolding this research is an exploration of the current narratives of time - of - arrival in cinema audio production and post production co ntexts . The impact of the 2020 Sound design solution is superimposed on these existing narratives, highlighting the intrinsic and extrinsic potentials of TOA alignment. These narratives are a synthesized amalgam of my own professional experience in the fie ld and the input of experts from audio production, post production, and engineering technician backgrounds. Figure 1 - Constructive and Destructive Interference of Two Summed Microphones Capturing a single source. The problem of time - of - arrival of multiple microphones negatively impacts the work I do as an audio professional. Every time I bring up two microphones in a mix, I lose as much as I gain when combining their images. They work against one another, softening and blurring t he potential that each microphone has on its own. This is a constant challenge, forcing me to use one microphone instead of multiple, only reaping the benefits of one perspective while sacrificing another. 2 While the issues of time - of - arrival are fixed and captured in production, they manifest in full during the post production process , limiting the mixers ability to leverage the combined potential of all microphones captured in production, often tying their hands to implement only one microphone in the fin al mix. Through this work’s exploration of the narratives of postproduction , the rationale for aligning time - of - arrival is revealed while highlighting the potential benefits of a solution . The novelty of these explorations lies not in the aim of aligning time - of - arrival but rather in its approach. This work acknowledge s existing techniques with parallel objective s as well their narrative in current production and post production workflows , evaluat ing the potential and limitations of these existing approaches . T his research explores the analogous problem of wow and flutter in mechanical analogue mediums as well as the digital latency inherent in complex audio systems . In the case of wow and flutter, t his examination of comparable temporal dist ortion s connects to a wealth of possible solutions that could be applied to the microphone time - of - arrival alignment problem. Contemporary designs that add focus and clarity to audio productions illustrates the context in which this work resides. While some desi gns examined do correct temporal distortions including time - of - arrival across multiple microphones, the common element of all examined designs is their interventions in the minutiae of perceived audio quality . While situating the work in a larger context, this exploration punctuates the extrinsic purpose of this work ; to play a role in improving the potential for clarity, focus, and precision in creating the next generation of soundtracks. This work stems from my experience in professional sound production and post production. Complimenting the professional experience is my passion for teaching audio production to the next generation of sound professionals. Th ese narrative s , in combination with a survey of experiences of the community of sound professionals , acts to define the problem while providing insight into its potential 3 solutions. An iterative approach to research through design fixes these narratives into the functional prototype produced. In developing the functional elements of the design, their po tential in the service of capturing, documenting, and aligning the position and temporality of sources and microphones were explored . This work looks into the technical building of the blocks for a solution that spans both production and post production contexts. Not all elements detailed are used directly in the finished designs as some components inspired the ultimate solution and others fulfilled the role of catalyst in exploring possible solutions. Beyond the technical underpinnings of the proposed solution, the user interaction and experience of this solution were explored from production and post production perspectives. How users connect to the potential of TOA alignment was examined alongside pot ential barriers and challenges that the design attempts to mitigate. U ltimately , this work culminates in a functional prototype of an ultrasonic clocking and digital signal processing time - of - arrival alignment system. The result i s a proof - of - concept proto type that lays the foundation for future iterations of the design. This acts as a foundation to further developed tools to align time - of - arrival with the ultimate goal of improving the perceived quality and impact of soundtracks. 4 2 Properties of Sound As thi s investigation explores the minutiae of sound in space, it would be remiss to not include a primer on the properties of sound that support and direct these investigations. From this framework of the fundamental properties of sound, the inherent issues of multiple microphone perspectives of a single source are revealed. This section then explores the challenges and potential of multiple microphone techniques in production and postproduction. With the problem identified, the analogous problem of wow and flut ter in analog recordings is examined for its parallels and possible insight into the time - of - arrival alignment problem at the core of this work. 2.1 Sound Fundamentals Sounds is objects vibrating in a medium. A vibrating object cyclically pushes and pulls on t he surrounding medium creating alternating states of compression and rarefaction. These compressions and rarefactions radiate outwards from the object in all directions. The magnitude of the compressions and rarefactions defines the amplitude of a sound. T heir rate defines the frequency of the sound. The speed of sound radiating away from the source is highly dependent on the medium itself. In our atmosphere, the speed of sound is generalized as 343 m/s 1 . This value assumes a temperature of 20 degrees Celsi us with the speed of sound increasing with temperature. The composition of the medium also plays a large role in the speed of transmission as, for example, sound travels 4.3 times faster in water than in air 2 . 1 "Speed of Sound." Wikipedia. March 08, 2019. Accessed March 18, 2019. https://en.wikipedia.org/wiki/Speed_of_sound. 2 "Speed of Sound." Wikipedia. March 08, 2019. Accessed March 18, 2019. https://en.wikipedia.org/wiki/Speed_of_sound. 5 2.2 Frequency and Wavelength Figure 2 - Amplitude, Phase , and Wavelength of Sound 3 The cycles of compression and rarefaction can be measured both in terms of time and space. The length of time it takes for a full cycle of compression and rarefaction is the frequency, measured in cycles per second or Hertz(Hz). The physical length of the cycle is documented as wavelength, measured in metres. As frequency increases , wavelength decreases and vice versa. This connection of time and space in the physical properties of sound are a fundamental component of the investigations of this project. At any given point in time, a sound’s current state of compression and rarefaction can be described as its phase, measured in degrees. The beginning of a cycle being 0 degrees and the comp letion of cycle documented as 360 degrees. 2.3 Time of Arrival As sound radiates from a source at the speed of sound, it arrives at objects in the acoustic space in order relative to the distance of each object to the source. A close perspective would have an earlier time - of - arrival relative to the later time - of - arrival of a more distant perspective. 3 Gary Dav is and Ralph Jones, The Sound Reinforcement Handbook , 2. ed., 2. printing (Milwaukee, Wis: Hal Leonard, 1990). 6 Figure 3 – Time - of - arrival of two sources with common signal 4 If a source is pushing against the atmosphere - creating a compression, th e time - of - arrival of that compression can be calculated by factoring the speed of sound with the distance of source to destination. If a source is 343 metres away from a destination, the sound arrives 1 second later. Put simply, for the purpose of this the sis, time - of - arrival is the amount of time for a sound to arrive at a destination from the source of sound itself. In this context, the destinations are microphones that transduce the acoustic energy into an electrical current, allowing for its recording. 2.4 Reflections As sound transverses an acoustic medium it is absorbed and reflected by the objects in its path. From the point of reflection, these reflection in turn then radiate within the acoustic space. Therefore, perception of a sound source is a combina tion of the direct sound from the source as well as reflections of the source. As the path of the reflected sound is not a direct line between source and ultimate point of perception, the time - of - flight is subsequently longer. At the point of perception, t he direct sound has 4 Francis Rumsey and Tim McCormick, Sound and Recording: Applications and Theory (O xford, UNITED KINGDOM: Taylor & Francis Group, 2014), http://ebookcentral.proquest.com/lib/ryerson/detail.action?docID=1638630. 7 an earlier time - of - arrival compared to that of the reflection. The rich confusion and softening of a sound source through reverberation is an analogous phenomenon to the summation of multiple microphones with varying times - of - arrival. 2.5 S umming multiple perspectives If summing multiple perspectives of a single source, the output of multiple microphones for example, the combination is at times additive or subtractive depending on the state of compression or rarefaction of each mic at any po int in time. One microphone may be transducing a compression whereas another transduces a state of compression. In this transduction, a compression translates to a positive voltage whereas a rarefaction results in a negative voltage. Whether a summation is additive or subtractive depends on the frequency or wavelength of the source and the distance between the microphones used in the summation. If the distance between perspectives is an exact multiple of the source’s wavelength, the summation is completely additive. The spaced pair of microphones in this case would be described as being in phase. A spacing that lies precisely halfway between on of these multiples would be totally subtractive , resulting in a summation result of zero or silence. Perspectives w ith this spacing would be considered out of phase. Points between in between these extremes lie on a gradient of additive and subtractive summation results. 8 Figure 4 - Summation of sine wave in varying phases 5 2.6 Comb filtering This talk of phase is overly simplistic in that it supposes sound sources vibrating at a single simple frequency, a sine wave. Practically speaking, the vibration of objects in space and the sound they generate is more complex. While a sound may have a fun damental tone, more often than not there are complex overtones and harmonics the prevent the sound from being described or documented as single frequency. Rather, a complex sound generates multiple frequencies or an entire spectra of frequencies. If a soun d source can be conceptualized to contain multiple frequencies, the additive and subtractive nature of the summation of multiple perspectives is much more complex. Across the entire frequency spectrum, the distance between perspectives relative to source h as a varying effect on whether the summation of a frequency is additive or subtractive. Alternating bands of the spectrum emerge as additive or subtractive, creating a comb - filtering effect. 5 Davis and Jones, The Sound Reinforcement Handbook . 9 Figure 5 – Comb - filtering resultant of two microphones at varying distances from source 2.7 Spatial Audio As ears are separated by the dimensions of the head, one ear leads the other in time - of - arrival of a source depending on which ear is closer to the source. This is one element that defines the head related transform function (HRTF) that allows humans to positionally locate a sound source in space. Other key factors of HRTF are the impacts the mass of the head shades the ears from the source in terms of amplitude and frequency response, the a ttenuation of frequencies depending on the head’s position relative to the ear. The core of this investigation probes the relationship of multiple microphone perspectives of singular sound sources. While there is a connection between this work and spatial audio, it is a tangential connection through aural perspective in real and virtual spaces. Spatial audio, in its attempts to create hyperreal reproductions for its audience attempts to map perspective of sound fields and sources to our two microphones, our ears. These two acoustic transducers are a spaced pair, but everyone’s spacing is unique. The holy grail of spatial audio research is translating the head - related transfer functions of one spatial experience to another. Suffice to say, position of microph ones, so to speak, is essential to the domain of spatial audio. It is an important distinction that this work is primarily concerned with relative distance between sources and microphones rather than absolute coordinates of a source in three - dimensional sp ace. 10 Nils Peters’ work explores systems of virtualizing acoustic space so that translations can be made to facilitate accurate HRTF experiences for the audience 6 . In this virtualized space, virtual microphones can be synthesized to capture the positional a nd directional perspective at particular coordinates in the virtual space. Peters’ investigations holds potentials to this thesis work as it endeavors to virtualize space in order to investigate possible solutions to align time - of - arrival of multiple micro phones. In audio, time is distance - tied to one another by the speed of sound. In altering the timing of multiple microphone perspective s , it’s changing their position in a virtual sense, creating a virtual world where alignment of microphones is just one possibility. 6 Nils Peters, “Sweet [Re]Production: Developing Sound Spatialization Tools for Musical Applications with Emphasis on Sweet Spot and off - Center Perception,” n.d., 305. 11 3 Time of Arrival Narratives Time - of - arrival is a fundamental component of the relationship between sound sources and perspectives. As such, it play a large role in the transduction, capture, and ultimate production of sound media. Its impact on these processes is unavoidable and, as a result, becomes a fundamental consideration of producers of sound media. This work explores the narratives of time - of - arrival from the perspective of audio production and post production perspectives . Through t he collection and synthesis of th ese narrative s , the rationale for intervention and ultimate alignment of time - of - arrival emerge s and become s apparent. These narratives stem from my professional experience and are scaffolded through interviews conducted wi th production and post production audio professionals in film and television. 3.1 Production At its core, the production scenario is the transduction of live audio sources in an acoustic space. While time - of - arrival is an important consideration in most audio production scenarios, the paradigm of film production sound production is the focus of this work. Regardless , there are parallels of this narrative to radio, television, music, and theatrical scenarios. In the film production sound scenario, the primary ob jective is to capture actors’ dialogue with as much presence, clarity, and focus as possible. Multiple microphones are used to capture the actors as they perform. A typical scenario could engage boom, body , and plan t microphones to capture a singular sourc e. The boom microphone is typically suspended above the subject, placed as close to the actor as the camera ’ s framing allows. The boom is a long arm that extends the reach of a boom operator, allowing th e placement of the microphone above the scene. The boom itself can range from a hand held pole to 12 a long fixed stationary pole with mechanical controls for movement. To isolate the subject from other sources in the scenario such as extraneous sounds, other subjects, or the acoustic of the space , the microphones used in boom operation are highly - directional. The boom operator’s challenge is to maximize proximity and point their microphone to wards the subject , all the while avoiding appearing in the camera’s f rame. This relationship between boom and camera creates a sympathetic match in perspective between the visual and aural capture. With a close - up visual composition, the microphone can be very close to the subject. With a wider shot, the boom must be furthe r away. The body microphone is a small microphone known as a lavaliere microphone. While a lavaliere can be affixed in a number of way s , the film production scenario requires it to be hidden on the body of the actor. As with the boom, the appearance of the microphone in - frame would break the diegetic narrative of the film by exposing the apparatus of the production. In this context, the lavaliere microphone is omni - directional. The microphone , therefore , can only be focused towards or away from a source thr ough its proximity. The body microphone is taped to the actor or integrated into their wardrobe. As such, the microphone ’ s perspective of the source is fixed resulting in a constant perspective that does not sympathetically parallel the visual shot composi tion as the boom microphone does. The body microphone, partially due to its size, is often a lesser quality microphone in comparison to a boom or plant microphone. To allow for movement in a scene and to prevent the cabling of the microphone from being see n by the camera, a wireless system connects the microphone to a recorder. This wireless transmission can only have a detrimental impact on quality when compared to a cable d boom or plant microphone. The positioning of the microphone, often on the actor’s s ternum underneath their clothing, is reflected in its perspective, a somewhat muffled perspective of a voice that is shaded by the chin. It is for these reasons of perspective and sonic fidelity that result in body microphones ’ consideration as secondary o r backup microphones to boom or plant microphones. 13 A plant microphone is similar to a boom microphone in many ways but is physically hidden in the scene rather than being suspended from above . These microphones can be planted in props or furniture that are proximate to an actor. In a car scenario, the microphone can be hidden in the car’s visor. In an office scene, the microphone c ould be hidden on the actors desk. Not all shots present opportunities for hiding microphones, but they can be a u seful complime ntary microphone to the boom and lavaliere . As fixed microphones, their perspective is dictated by their position and directionality and do not respond to the changing visual frame of the camera nor the movement of the actor . It is through this implementat ion of multiple microphones of a single source that issues of time - of - arrival arise. If two microphones captur e they same source, the time - of - arrival of that source will be misaligned between the microphones. The closer microphone transduces the compressio ns and rarefactions emanating from the sound source first, with the transduction of the same compressions and rarefactions happening later in more distant microphones. It is this scenario that is the primary focus of this work : the misalignment of multiple microphones due to the relative differences in distance between a singular source and multiple microphones. This misalignment is apparent in the mix created by the production sound mix operator. As they combine the misaligned images of multiple microphone s into a production mix, the mix loses focus, clarity, and p re cision. 3.1.1 Two Sources, one perspective There are, however, other time - of - arrival scenarios that impact the production process. While this work focusses on the scenario of a single source perceived at multiple perspectives, time - of - arrival has the same effect in the inverse scenario of multiple matched sources arriving at a single perspective. This inverse problem has parallel nar ratives in production and post production . 14 As an example, take the aural experience of a boom operator. They extend a microphone with a boom, placing the microphone closer to the source relative to their ears. T he boom operator perceives a confused image t hat is the blend of the earlier arrival of the boom microphone and the later arrival of the same source at their ears. 3.2 Post Production Post production in this context refers to the audio editorial and mix processes that create the final soundtrack for a fi lm. These processes occur after the narrative video editorial process where audio and visual material captured in production is selected, edited, and sequenced to create the fixed linear form referred to as locked picture. As the focus of this work is the alignment of multiple microphones, this section focus es on the dialogue editorial and mix processes specifically. Generally, dialogue is the primary consideration and focus of capture in production ; other elements such as sound effects and music are added entirely in the post production process. Time - of - arrival is not a primary consideration of the dialogue editorial team unless they endeavour to correct or align using manual methods detailed in subsequent section s . The focus of the dialogue editorial proce ss is to finesse and augment the dialogue cut from the narrative video editorial process. In evaluating the dialogue, the dialogue editorial team smooth s the edits of the video editor and, if it cannot be finessed, replace s the selected line with another t ake or performance from production. If no fix can be found, the actor is reengaged to reperform the line in a studio as a replacement to the original dialogue in a process known as ADR. Extraneous noise and sounds contaminating a recording or an issue of t alent performance are common obstacles that require a line to be rerecorded and replace d . The dialogue editorial finesse s and deliver s edits of all microphone types recorded in production, leaving the decision of which microphone to implement to the downst ream mix process. Even in rerecording 15 performances through ADR, multiple microphones are captured to mimic the perspectives recorded in production, thus allowing for a more seamless match between production and replaced dialogue recordings. It is in the mi x process that time - of - arrival becomes apparent as an issue . As all microphones are finessed and passed from the editorial to mix stage, the mixer has options as to which microphones contribute to the final soundtrack. A mixer combining multiple microphone s is presented with the challenge of combining multiple perspectives with varying time - of - arrival . As outlined earlier, t he summation of multiple microphones on a single source is not wholly additive, nor wholly subtractive. As the microphones blend with o ne another, common characteristics of the two perspectives are emphasized and differences are deemphasized . The time - of - arrival differences between the two microphones combine to create a blurry, smeared image with less focus and fidelity than that of each discrete perspective. With the problematic reality of mixing misaligned microphone positions, the most precise option for mix clarity and focus is to use only one microphone. This removes any lack of clarity introduced by mixing multiple microphones. In this case, the boom micr ophone would be the ideal choice with its sympathetic match between the visual and aural perspectives. If the boom is not a feasible choice, say for reasons of poor or inconsistent boom technique, other mics may be selected as the sole contributor to the m ix. However, with either body or plant microphones, aural perspective would have be matched to the visual through labour - intensive processing and automation. Otherwise, the sound doesn’t respond to the visual perspective and feels disconnected as a result. 3.2.1 Time of Arrival Alignment Aligning time - of - arrival restore s the potential of contributing multiple microphones without suffering a loss in fidelity in doing so. With aligned microphone perspectives, a mixer is able to blend perspectives 16 as desired. For ex ample, the shifting perspective of the boom could be ground ed and reinforced with the consistency a lavaliere . Two microphones that before worked against one another would become more complimentary. The sonic potential of mixes would be expanded as they wo uld allow for the contribution of elements from multiple perspectives rather than that of a single microphone. 3.2.1.1 Current Methods These benefits are not the stuff of theoretical abstraction. A m ethod of alignment for multiple microphones exist s and is impleme nted in productions that have the expertise and capacity to implement it . However, this method is manual and, as a result, is time and labour intensive. As a result, it is not a commonplace practice . Figure 6 - Dialogue Editorial Alignment 7 7 John Purcell, Dialogue Editing for Motion Pictures : A Guide to the Invisible Art (Routledge, 2013), https://doi.org/10.4324/9780203784570. 17 To align perspectives of multiple microphones, the recor dings are manually cut and aligned in editorial. By eye, an editor looks at the cycles of compression and rarefaction and advances or delays them in order to align them. This alignment is static and does not allow for the dynamic alignment of moving microp hones or sources. The resolution of the alignment increases with each additional cut and alignment but can never fully align against continuous movement. 3.3 Analogous Problems 3.3.1 Issues in Music In music production, it is common to implement multiple microphones to capture a single source or a combination of sources. Take for example, contemporary practices of capturing a drum kit. 3.3.1.1 Capturing Drums While a single microphone captures the drum kit overall , it does not allow for the discrete rebalancing of drums post - production in the mix. A kick drum cannot be raised in amplitude relative to a tom drum, for example. As such, individual microphones are used to capture each individual drum as discretely as possible. While possible to create a drum mix using only these discrete perspectives, aesthetically this mix lacks the cohesion of the kit as a whole. A compromise is struck where overhead microphones capture the drum kit as a whole , which is reinforced with individual spot mics on individual drums. The perspective of this approach can treated either way in the mix as the mixer can select which microphone type, spot , or overhead form the basis of the mix and wh ich type acts as reinforcement. As the spot microphones are relatively closer to each individual drum, their time - of - arrival is earlier than that of the overhead microphones. Mixing the two perspectives has the overhead microphones behind the spot micropho nes, softening the transient impact of the percussive drum strikes. Striking a drum is a highly percussive sound with a strong transient or sudden change in sound pressure level. It’s worth noting that this softening of the drums can be an aesthetic choice in its own right. 18 A common strategy to align the time - of - arrival between overhead and spot microphones is to delay the playback of the spot microphone recordings in the mix. In delaying the spot microphones , they are effectively moved in space, delaying their time - of - arrival . Through this method, the time - of - arrival of spot microphones can be aligned to that of the overheads. This approach is only effective when delaying the spot microphones in that, as with their amplitude, individual drums cannot be retimed in the overhead microphone image. By aligning time - of - arrival in the mix, a punchier drum sound is created through a single non - diffused transient. With or without this alignment, it a common approach to gate drum microphones . The processing of gating effectively turn s a microphone off if its target drum is not in u se at a given time. By turning microphones off when they are not in direct use, the y do not contribute off - microphone bleed to the mix. For example if someone is playing the snare and not the tom, the tom microphone turns off, cancelling the downstream tim e - of - arrival perspective of the snare on the tom microphone. Through this technique, the mix is made more coherent by having the fewest possible microphones contribute to the mix. Another approach is to mitigate the issue of time - of - arrival in production. The close microphone approach relies on several microphones. Techniques that use fewer microphones naturally have fewer intersections and conflicts of time - of - arrival . A common technique is to use three microphones that are equidistant to the centre of the drum kit, the snare. Th is results in recordings that are fundamentally more aligned than the overhead and close microphone approach. While it is not the explicit goal of this work to align audio in a musical context, the proposed design would accommodate this scenario, resulting in mixes that retain more of the transient punch of their acoustic sources. 19 3.3.2 Wow and Flutter in Analog Recordings The theoretical solution and prototype for the analogous problem of wow and flutter on vintage recordings holds potent ial in the investigation into time - of - arrival alignment of multiple microphones. Many old recordings are warped; they speed up and slow down, pitching up and down as a result. Recording audio is a process of capturing a dynamic electronic signal that docu ments a musician, speaker, or whatever source occupies the content dimension of the recorder’s input. This material is recorded against time, correlated against the recording medium as a transport rate. For example, analogue tape would specify its transpor t rate in inches per second, connecting time to the medium. Each recording and playback device relies on a clock to drive this rate. For example, your turntable may have a quartz clock circuit that drives the physical turning of a platter. Ideally, a devic e’s clock would be perfect and material recorded and subsequently played - back on that machine would perfectly reflect the temporality of the recording scenario. However, these clocks and the devices they drive are not perfect. All audio devices deviate fro m their specified transport rate to varying degrees. Older mechanical analogue devices such as tape and vinyl suffer from wow and flutter, the relatively low and high rate audible deviations of transport rate respectively. Contemporary devices suffer from jitter, variations of a higher - order that while not directly audible, compromise the quality of an audio recording. If a recorder device suffered from noticeable wow and flutter in recording, it’s baked - in to the recording, manifesting as the inverse devia tion on playback. For example, if a recorder transport rate slowed during recording, there would be an audible increase in rate and pitch during that segment. In vinyl recordings, the clock of the original recording could be distorted with misaligned centr e hole. This temporal distortion has parallels to the doppler effect perceived as a microphone moves closer relative to a source or vice versa. In the case of analog recordings, the modulation is the result of a bad 20 clock that drives a mechanical transport mechanism whereas the distortions investigated in this work are the result of changes in distance and the propagation of sound waves at the speed of sound. In one phase of their work to correct wow and flutter, Wallaszkovits et al leverage a hidden consta nt buried in the analogue tape recordings containing wow and flutter 8 . The recordings contain a high - frequency bias tone that resides in the inaudible ultrasonic frequency domain. This tone, created by an oscillator (clock) at the genesis of the recording, originally had a consistent pitch. Recorded to tape, this tone is no longer constant but suffers from the same wow and flutter of the audible recorded material. Wallaskovits et al implement a process to restore this fluctuating pitch to a consistent pitch and, in the process, remove the wow and flutter from the recording as a whole. Figure 7 - Pitch variation: uncorrected (left) and corrected (right) 9 If a hidden clock can be recorded alongside normal content, hidden from the au dience, then this clock could be used to document TOA and facilitate its correction. In Wallaskovitz et al work, the figurative high - frequency bias bread crumbs that allowed them to return recordings to a flat temporality were an unintentional consequence of the original recording technology implemented. Intentionally creating and documenting a similar inaudible frequency provides purchase to align multiple moving microphone perspectives to a source. 8 Nadja Wallaszkovits, Tobias Hetzer, and Heinrich Pichler, “Drift and Wow Correction of Analogue Magnetic Tape Recordings in the Analogue Domain Using HF - Bias Signals,” in Audio Engineering Society Convention 136 , 2014, http://www.aes.org/ e - lib/browse.cfm?elib=17189. 9 Wallaszkovits, Hetzer, and Pichler. 21 Walloskovitz et al are not the first or only to investiga te the flattening of wow and flutter. Andrzej Ciarkowski and Andrzej Czyzewski have an extensive body of work in the pursuit of correcting warped analog recordings. In their work, low frequency hum present in many analog recordings presents a temporal guide for restoration 10 . While on opposite ends of the audible spectrum, the princi ple is similar; using an artifact from the recording process as way to correct pitch deviations. 3.3.3 Audio System Latencies Complex audio systems suffer from the impact of computational latency. This latency is the result of digital processes that digitize, bu ffer, and process audio . Each process takes time and therefore adds an offset or delay to the signals passing through them. Contemporary digital audio systems compensate for their internal delay , ensuring that , while there is an overall delay, all signals are delayed equally regardless of the delay required for each signal. In Avid’s Pro Tools for example, all signals are delayed to match the longest or most latent processing path in a process known as delay compensation . In playback, this delay is a non - is sue as the recordings that feed playback for processing are advanced to negate the delay. It is in larger interconnected digital audio systems with multiple independent devices that this problem emerges on a larger scale. Each device has its own latency as sociated with it. As the systems are unique and often proprietary , there is little hope of delay compensation being implemented in an effective way. This misalignment is not necessarily static either . As sample rate convertors allows unclocked digital devi ces to be connected to one another, they are allowing for the interconnection of devices that are running at fundamentally different rate s . This creates dynamic temporal distortions between devices connected in this fashion. 10 Andrzej Ciarkowski et al., “Methods for Detection and Removal of Parasitic Frequency Modulation in Audio Recordings,” in Audio Engineering Society Conference: 26th International Conference: Audio Forensics in the Digital Age , 2005, http://www.ae s.org/e - lib/browse.cfm?elib=13246. 22 While I do not foresee the pote ntial of this work to remedy this issue in real - time, the proposed method of inserting an ultrasonic clock in - band, married to the audio, could allow for the realignment of audio that has travelled through disparate latent audio paths. 23 4 Methodology Through my professional experience and countless engagements in scenarios involving issues of time - of - arrival , I resolved to explore solutions to the issue. The proposed solution of a high - resolution in - band clocking signal emanating from a source giving purchase for an algorithm to defeat the temporal distortion and ultimately the positional movement of microphones came fairly quickly in these early ruminations. This quickly became the goal ; to explore if this approach was feasible. If LTC could be recorded in - ban d and used to restore the temporality of parallel recordings, surely an ultrasonic signal recorded alongside audible information could lead to similar solutions. With these early visions of what could be possible, the subsequent work followed this vision i n an attempt to realize and implement its potential. I did not begin with a particular methodology in place before exploring the domain of audio production and identifying issues within current practices of the field. A bottom up would reproach would start with a methodological foundation, allowing for an emergent yet unknown project to unfold. I saw this neither as an advantage nor disadvantage as , while I sidestepped the challenges of finding my project through exploration, I had the challenge of plotting a course towards a particular frame of desired outcomes. With these beginnings, my explorations of methodology were directed at scaffolding my vision and plotting a course towards my desired outcomes. This work eschews a formal academic design methodology . This partially due to the top - down discovery of the problem and solution coming before engaging with a formal thesis process. A methodology is an approach. While it would be disingenuous to align this work under the umbrella of a complete design methodol ogy, the follow ing are components of the overall framework used in completing this work. 24 4.1 Narrative - Driven User - Centred Design Users play a central role both in the investigational vectors plotted by my research questions as well as my prototype goals of cr eating a useable time - of - arrival alignment tool. In formulating my initial inquiries, I too was a user that experienced the issues of TOA and could possibility benefit from its alignment. The work of this thesis comes from the observation of a user’s need and ultimately aims to engage users with the results of this investigation. As such, this thesis incorporates elements of user centred design (UCD) in its design methodology. Rather than use standard ethnographic approaches within the UCD, I have opted to instead implement a narrative inquiry approach. The approach of Gausepohl et al uses a storytelling protocol that collects narratives from both stakeholders and designers 11 . A process of narrative analysis follows where ‘user needs are explored and design opportunities are identified.’ The narrative process allows for tacit and explicit knowledge to identify these user needs, facilitating a design process that moves from the ‘problem space’ to the ‘solution space’. This project engage s a storytelling protocol similar to that proposed by Gausepohl et al. By interviewing experts in audio production and engineering, I have analyze d individual narratives to synthesize common narr atives that illuminate user needs in the problem space, allowing for my design to create solutions to address those needs. 4.2 Research through Design My thesis work is an example of research - through - design as it endeavours to both create a specific solution applied to the world through design as well as extending knowledge within the field through 11 A. Gausepohl , Kimberly, Woodrow W. Winchester, Tonya L. Smith - Jackson, Brian M. Kleiner, and James D. Arthur. “A Conceptual Model for the Role of Storytelling in Design: Leveraging Narrative Inquiry in User - Centered Design (UCD).” Health and Technology 6, no. 2 (July 2016): 125 – 36. https://doi.org/10.1007/s12553 - 015 - 0123 - 1 . 25 research . 12 As research, th is thesis provides contributio ns of abstracted theory and, as design, produces a situated realization of a prototype. It is an important distinction that this thesis is research through design rather than research for design. The design component propels the research component rather t han the research scaffolding the design. Through my work, I am probing the issue of time - of - arrival of a source at multiple microphones. My prototype design work is intended to examine that issue. While the lines between research and design blurs as they p ropel one another forward, it is this kernel of intent that frames how I approach the theoretical writing and prototypical making. Both frameworks of research for design and research through design are useful as lenses to unpack and reveal what lessons spe cific designs could have in a larger theoretical context as well the inverse, revealing the potential general abstract theory can have in my specifically situated design solutions. 4.3 Iterative Prototyp ing This work endeavour s to create a working prototype of a system that will correct for differences in time - of - arrival between microphone sources. In the scope allowable in this thesis, the prototype will exist virtually, moving ‘outside the box’ only after satisfactory completion of a virtual prototype. With t he ultimate design conceived of before this thesis approach, the process of making the prototype is an iterative exploration and execution of methods and technology in order to reach the ultimate goal of time - of - arrival alignment. An example of the iterati ve making is the execution of a clock interpolation engine. The earliest iterations of this implement a crude parallel ‘dummy clock’ that is a stand in for the eventual in - band 12 Mads Soegaard and Rikke Friis Dam. 2013. The Encyclopedia of Human - Computer Interaction , 2nd Ed. (2nd ed.). The Interaction Design Founda tion. 26 ultrasonic clock. Through multiple successful iterations and countless failures , this algorithm evolved to meet the specific demands of the overarching design goals. 4.4 Learning by Teaching This work is borne from my years as a professor of media production arts at Ryerson University. In fact, the initial concept was sparked in my inter mediate audio class. In this class, students make the leap from recording sources with a single microphone to capturing complex sources with multiple microphones. This exercise focuses on the principles and considerations of combining the perspectives of m ultiple microphones, highlighting the benefits and challenges of moving beyond a single microphone. It was at this moment that the concept of positionally - aware microphones came to me. Years later , in engaging in a formal thesis process with a cohort from a wide range of disciplines and experiences, I quickly realized and was informed that my thesis would require attention to not only the intrinsic challenge of time - of - arrival alignment but also to the extrinsic challenge of connecting my work to a larger audience outside the domain audio production professionals. By connecting these ideas with a general audience, it forced me to better connect with the core principles of the problem and its solution , let alone the rationale for solution. Through this work, I hope to document, illustrate, and present on this topic on an ongoing basis. By connecting the work to others, I continue to explore and refine my own understanding of it. 27 5 Existing Designs As this work and resultant prototype will manifest as a profes sional production tool, the bulk of compatible projects examined fall into that classification. In the world of professional audio production tools, there are a number of designs that reflect elements of this work. There are many tools that finesse the time, phase, and pitch of audio; even tools that work at the same level of precision that this work aims for. 5.1 Celemony Capstan Autotune ushered in a new era of music production; the pitch of any performer could be tweaked. Cher’s Life after Love is the most obvious example of purposefully egregious autotuning, but countless tunes have slight deviations in pitch seamlessly corrected for good or ill. Can’t sing? No problem! Want to sou nd like a warbly robot? Also, not a problem. Antares’ Autotune and Celemony’s Melodyne are still the leaders of this technology although many newer players have entered the space; the tools have become quite commonplace, available to professionals and novi ces alike. Antares took an automatic set - and - forget approach to correcting pitch, whereas Melodyne provided advanced editing tools to allow for custom pitch - design of a performance using their patented DNA technology. In 2012 Celemony directed this technol ogy for correcting vocal pitch towards fixing warped recordings in a software release titled Capstan 13 . Capstan is a commercially available audio processor that corrects wow and flutter pitch deviations present in recordings. It attempts to realign recordi ngs with their original temporal reality and, in doing so, their original rate and pitch. Capstan analyzes the original recording, determining instances within the recording of wow and flutter. The software is looking for global changes in pitch in the rec ording 13 “Celemony | Capstan,” accessed November 22, 2018, https://www.celemony.com/en/capstan. 28 that are indicative of a recording or transfer timing issue. With the wow and flutter identified, the recording is varisped at a rate inverse to the original deviation, thereby removing it. Advanced settings and editorial tools allow the user to dis tinguish between intended pitch deviation (vibrato) and the problematic wow & flutter. Users are also able to finesse and fine - tune the process of correcting for the wow & flutter. This technology has the capacity to resurrect a wealth of recordings and co nnect them with a new audience. The quality issues of older technology often limit the connection a performance can make with a contemporary audience not able to hear beyond the effect of wow and flutter. By realigning the recording with real - time, the vei l of technological error and perspective is lifted, making a stronger connection between the artist and audience. This is the ultimate conceptual aim of this thesis to make a stronger and more transparent connection between artist and audience. On a techni cal level, the dynamic, automatic, changes in pitch (varispeed chase) that Capstan provides connects it to the investigations of this thesis. 5.2 Dan Dugan Automixer The life of a production audio mixer can be stressful business. Take your typical roundtable d iscussion scenario on the nightly news as an example. It’s unscripted, anyone can contribute at any time; sometimes two panelists are talking over one another. Leave all the microphones up in the mix and the mix sounds unfocused and ambient; the mix lacks definition and fidelity. The current speaker is present not only on their own mic; their voice also contributes to everyone else's mic. The sound of their voice arrives at their own, relatively proximate, mic rophone first before arriving at all other micro phones. The other microphones are also more distant with a higher ratio of ambience to voice present in their off - mic contribution. These off - mic qualities fight the speaker’s own mic rophone in the mix, detracting from the quality and precision of the mix. 29 The Dan Dugan Automixer 14 is a commercial audio tool that automatically mixes multiple microphone scenarios. Using a proprietary process, the Dugan compares the relative level of incoming signals. The strongest signal is passed while the relatively lower s ignals are attenuated. If one person is talking, they contribute to their own microphone primarily and bleed into other microphones at much lower level. The Dugan allows the highest signal to pass, attenuating all other sources, diminishing them in the mix . If two channels in the system have an active contribution, the Dugan allows both signals to pass, but attenuates them both reduce the level of two people speaking over one another. Figure 8 - Dan Dugan Automixer for Waves Multirack 15 The Dugan defines the contemporary epoch of broadcast production audio quality. This simple yet effective process allows for more dynamic and unscripted oral scenarios to be reproduced in a range of mediums. The communication potentia l of hearing a large freeform group as a whole and as individuals is made possible by the Dugan Automixer. As such, it enlarges the scope of possible forms of engagement between artist and audience while simultaneously making a stronger connection between artist and audience through enhanced clarity. 14 “Dugan Automatic Microphone Mixers,” accessed November 1, 2018, https://www.dandugan.com/products/. 15 “Dugan Automatic Microphone Mixers.” 30 Dan Dugan’s work relates to this thesis in the clarity and focus it brings to complex multi - microphone situations. In a tangential fashion, the Dugan addresses time - of - arrival issues by simply removing multiple microphones from the mixed image, a blunt but effective approach. 5.3 Syncro Arts VocAlign Issues arise in production audio scenarios that compromise the quality of the recorded audio. Great performance, but an airplane flew overhead. Great delivery, but some thing was wrong with your microphone. Nice acting, but there’s no way we could get useable audio with all the pyrotechnics. Great recording, but your performance was lacking. Fear not. Ambient, technical, and performance issues can be solved by replacing s poken dialogue with new recordings. ADR is the agreed upon term for replacing production dialogue with dubs. It is standard practice to replace a varying amount of dialogue in a narrative film or television production for either technical or performance re asons. It’s a big ask to get actors to recreate the emotion and action of a scene out - of - the - moment. You’re no longer on a ship on wild seas fighting dragons, you’re in a comfortably lit sound room with croissants. On top of that, you have to replace lines with the exact same timing as the original. If not, it can easily look like a bad foreign language dub a - la - Godzilla. With beeps in your headphones and wipes on the screen, the recordist tries to help you with your sync as much as possible, but it’s still really challenging. ADR is often out of sync with the audio it’s trying to replace. Matched with the original visuals, this detracts from the potential connection of the audience to the diegetic narrative world created. VocAlign 16 is a commercial audio pro cessor that synchronizes replacement dialogue (ADR) to the timing of the recording it replaces creating a perfect match between ADR and the visual. VocAlign scans both 16 “VocALign PRO 4 - Overview - Synchro Arts,” accessed November 18, 2018, https://www.synchroarts.com/products/vocalign - pro/overview. 31 original production audio and replacement audio. The aural content and cadence of the pe rformance are analyzed and mapped temporally. VocAlign then morphs the replacement audio to match the original. Figure 9 - VocAlign PRO 4 17 VocAlign allows performers and artists to make a stronger connection with their audio thro ugh the removal of timing inconsistencies between the aural and the visual. Out - of - sync dialogue can be very distracting and detracting from the perceived quality of a piece. Bad sync reveals the apparatus of film making to the audience, breaking down the fourth wall; compromising the narrative contract between artist and audience. This concept of temporal correction connects with the explorations and intent of this thesis. While the offset and variations of pitch of moving microphones are not of the magnit ude that would resonate as a lip sync error, the morphing from one temporal framework to another provides insight into these investigations. 17 “VocALign PRO 4 - Overview - Synchro Arts.” 32 5.4 Sound Radix Auto Align Post Sound Radix’s Auto Align Post 18 is a post - production audio processing tool that promises to align time - of - arrival between microphone. The process analyzes a recording and processes other recordings to align their time - of - arrival . This process fixes the time - of - arrival of multiple microphones to that of a single microphone. While the overall o bjectives of Auto Align Post and this investigation do have parallels, there are key differences in its technical methods and outcomes. Both endeavor to align time - of - arrival across multiple microphones. However, without a reference to temporality from pr oduction, Auto Align Post is aligning the time - of - arrival between microphones and not to a designated source. Figure 10 - Sound Radix Auto Align Post 19 Through the proposed methods of this thesis, the prototyped solution will not o nly align microphones to one another but also to the source. The microphones are aligned and fixed relative to the source. There will be no perceived motion towards or away from a tracked source. In Auto Align Post, if the master recording moves towards aw ay from a source, all microphones will follow suit and move in tandem. The 18 “Auto - Align Post: The Fast & Simple Way to Get Location Mics in Phase ,” Sound Radix, accessed November 2, 2018, https://www.soundradix.com/products/auto - align - post/. 19 “Auto - Align Post.” 33 untracked and uncorrected variable distance between microphones and source is a key differentiator between Auto Align Post and this investigation. Another key difference is the impl emented temporal correction method . Due to its proprietary nature, the exact method implemented by Auto Align Post is unknown. However, as no production temporal recording or metadata is required, the timing and subsequent processing are approximated by an analysis algorithm of unknow design. While this adds a la yer of convenience as no extra measures are required in production, there is potential for misinterpretation and mis alignment without a production temporal reference. As the release of Auto Align Post is a recent development, its impact on dialogue mixes a nd overall soundtrack quality has yet to emerge. 34 6 2020 Sound 6.1 Conceptual Framework While the problem of time - of - arrival manifests in post - production, the issue stems from the physical realities of time and space in production or initial recording itself. I n the context of this project, production refers to the production scenario wherein sources are captured with microphones. The composite imaging problem of TOA is the result of relative differences in TOA of each microphone relative to the source. Documenting the relative position of the source to each microphone would provide ‘breadcrumbs’ that could be used to align the microphones in post production. The time - of - arrival of a source at each microphone is visually apparent when inspecting the wavef orms of their recordings. Their respective time - of - arrival of the source is baked in to the recording itself. If relatively closer, the source is recorded relatively earlier and vice versa. If the source emitted an aural clock signal , that could be used as a reference for the source’s time - of - arrival at each microphone. An audible clock cannot be dismissed out - of - hand. An audible pulse of a single frequency could be removed after the fact using a notch filter to eliminate the frequency of the clock. However , this clock would be a nuisance in production, distracting members of the production, actors delivering lines for example. A n ultrasonic clock above the human audible frequency range of 20Hz to 20,000Hz could be used to silently document the relative loca tion of a source. While we cannot hear anything above 20000 Hz , contemporary audio equipment including microphones and recorders are capable of recording ultrasonic frequencies. 35 6.2 Production Figure 11 - Production Design Overview In production, an ultrasonic beacon that emits a clock pulse would be placed at the sou rce to be tracked. It is unacceptable to see microphones in narrative film productions, so the same could be assumed of any other technology that is out - of - place in the di egetic space. The beacon would need to be hidden from the camera at a static distance from the source. For the purposes of this project the mouth is the positional reference to the sound source of an actor. H iding the beacon on the sternum of the actor wou ld provide a relatively static position in relation to the source. This compliments a common placement for lavaliere microphones as , for male and female alike, the sternum often affords a small cavity in which to hide the microphone underneath clothing. On ce powered, the beacon constantly emits the reference clock signal. In future designs, beacons would be assigned a unique frequency to allow for multiple beacons to track multiple actors in a scene. The beacon is the only novel or additional piece of physi cal hardware demanded by this design. However, in order to transduce and capture the ultrasonic clocking beacon, the microphone and recording apparatus used would have to be able to transduce and record at those frequencies. While on the esoteric end of the microphone spectrum, there are film microphones such as the Schoeps MK41 capsule with CMC6xt microphone amplifier comb ination that are able capture up to 40kHz . However, while this capsule is fairly common in professional production, the extend ed range amplifier is not. 36 Lavaliere microphones are a different story with a common frequency response ceiling of 20kHz, below the ultrasonic range. This problem is compounded by wireless transmission systems used to connect recorders to microphones wirel essly with even the newest generation of wireless transmitters topping out at a ceiling of 20kHz. While this does not bode well for ultrasonic clocking beacon to lavaliere compatibility , it is not a severe setback to the design. The beacon would be placed at or near a lavaliere or, at the very least, in a static position relative to the source. As a result, the lavaliere is already closely aligned to the source. If other microphones, say plant or boom, were aligned to the beacon they would become aligned to the lavaliere as a result. Contemporary field recorders are designed to record digital at sample rate of 96000kHz, making them compatible with an ultrasonic beacon by facilitating the recording of signals as high as 48000kHz as per the Nyquist - Shannon sam pling theorem 20 which dictates that a sampling rate of x can reproduce a maximum frequency of x/2. Beyond these technical consideration, the production process proceeds as usual albeit with the addition of ultrasonic beacons fixed to sources. This fulfills a goal of the design, to require minimal intervention or disruption in the current workflow of contemporary productions. The design has potential benefits for productions that make use of it, but does not fundamentally change the equipment required or proc ess fulfilled in production. If, for whatever reason, the ultrasonic beacon fails in production, the production audio is still viable in a traditional sense , lacking only the ability to be aligned automatically in the post production component of the desig n. 20 "Nyquist – Shannon Sampling Theorem." Wikipedia. March 01, 2019. Accessed March 18, 2019. https://en.wikipedia.org/wiki/Nyquist – Shannon_sampling_theorem. 37 6.3 Post Production Figure 12 - Post Production Design Overview The design solution for the post production time - of - arrival tool takes the form of an offline audio processor plug - in. As with most time correction utilities, the ver y nature of its time manipulation process prevents it from operating in real - time. Ultimately, this would take the form of a Pro Tools Audiosuite, VST, and Audio Units compatible processor allowing for its application in a wide range of digital audio works tation (DAW) environments. The processor relies on audio recordings with a common temporal and acoustic scenario; that they were record ed in the same time and space. Production recording devices document this information in metadata referring to the shot a nd take number, even if only in the filename. If implemented correctly in production, the recordings also contain timecode metadata that documents the time of the recording. Figure 13 - User and Process Flow The user selects a se t of compatible recordings and loads the DSP time - alignment processor. These recordings are then displayed visually as waveforms. The processor anal yses the recording set for valid 38 ultrasonic clock pulses and indicates the results by highlighting the waveforms that comply . Only recordings that have valid ultrasonic clock information will ultimately be processed. As discussed in the production design, lavalieres will not capture the ultrasonic clock signal but are already aligned to the source. The deviation between recordings is displayed in a graph. This demonstrates the magnitude of misalignment between the recordings. The user can choose to dynamically or statically align the recordings. A static alignment aligns a selected point in time across all recordings, but does not dynamic correct for movement throughout the recordings. The processor defaults to aligning the top of the recordings if no point is specified in static alignment. This choice allows for a simpler, less invasive alig nment of recordings; suitable for microphones and sources that do not move during a recording. This is an ideal setting for aligning microphones on a drum kit, for example. This option would be no different than the manual editorial process . The dynamic al ignment process aligns the microphones throughout duration of the recording, virtually negating their movement. Figure 14 - User Interface 39 With the alignment type selected, the alignment can be previewed, playing from directly th rough the interface after the processor computes the alignment. If satisfactory, the alignment can be committed and returned in - place into the DAW environment from whence it came. The integration will vary depending on the DAW and processor plug - in format but , while the processed alignment will return in - place versions of the original unaligned audio, it will not destructively overwrite or replace the original recordings . R ather , it will create new processed files. This is standard practice for offline audi o processors. The user can proceed and continue working with the processed clips in their DAW as they would the original recordings. 6.4 User Narrative Design As part of my design process, I sketched out the user experience for my prototype design in narrative form. These narratives are synthesized, based on my own experiences in addition to experts interviewed as part of my narrative research. C reating functional narratives for th e intended design was invaluable for identify the key components and user flow of the ultimate design. 6.4.1 Production Jan is a production sound mixer for narrative & documentary film and television productions. She owns and maintains a recording kit that incl udes lavaliere , shotgun, and small diaphragm condenser microphones. These microphones track subjects in a scene as body, boom, planted, and boundary microphones. In addition to affixing body mics to her subjects, she discreetly places the 2020 Sound locali zation beacons to each subject she wants to track. She double checks that each beacon is on and has a unique address assigned to distinguish each beacon. Jan spec’d her microphones, wireless transmitters, and recorder to ensure they can record the ultrason ic clock pulses of the beacons. Jan 40 records ISOs and mixes of each scene as per usual, delivering them to the production for use in offline picture editorial and dialogue editorial. 6.4.2 Post P roduction After receiving a locked cut of a film, Krystin begins the dialogue editorial process. As usual, she cuts dialogue from the source ISO recordings to sync with picture and cuts in alternate takes as required. After the production dialogue is edited, patched & finessed, Krystin launches the 2020 sound offline proce ssor. Krystin selects multiple clips that belong to the same temporal recording set. She clicks 'Analyze’ and 2020 begins scanning the selected clips for a valid ultrasonic positional clock signal. If valid, Krystin can select which source she wants to ali gn to. For clips with multiple actors, Krystin assigns body mics directly to a beacon. Depending on who’s talking at any given point, 2020 dynamically changes its alignment to focus on the actor currently speaking. Krystin, previews the alignment, tweaking the fine tuning settings as needed. Satisfied with the alignment, Krystin clicks 'Commit’, rendering aligned audio to new clips on a new editorial playlist. 41 7 Prototyping The prototype explored in this thesis is a virtual proof - of - concept for both an in - ban d ultrasonic clock for use in production as well as a DSP time - of - arrival alignment algorithm. While this does not progress as far as real microphones in an acoustic space, it forms the basis of the technical underpinnings required to make this design function . In this context, the audio source is somewhat irrelevant as an exis ting recording can stand in for a source. The microphone perspectives are affected translations of the source material. By adding a unique initial temporal offset (delay) to each copy of the source, distance relative to the unaffected reference source is c reated. By uniquely modulating the temporal rate of the files, the movement of microphones throughout the recording is synthesized. While these process es exist virtually outside of an acoustic space, they mimic the offset of perspectives and their movement . In attempting to align these perspectives back to the reference source (the original temporality), the challenge becomes reversing the processes that make the perspectives unique and dynamically changing. The roadmap to achieving this begins in the ultra sonic clock. The ultrasonic clock is married to the sonic material before the material is cloned, offset, and modulated temporally. It is this clock that creates the ‘breadcrumbs’ that allow us to find our way back the temporality of the reference audio. 7.1 U ltrasonic Clocking Generating a reference tone above the audible spectrum proved a simple task. Audacity was able to generate a tone of any frequency allowable by the format of audio file . The sampling rate, the number of amplitude samples recorded per sec ond, dictates the maximum frequency the recording can capture or reproduce. The Nyquist theorem states that a sampling rate of x can reproduce a maximum frequency of x /2. The common sampling rate of 48000Hz therefore can reproduce frequencies up to 24000Hz . While this did leave some room to generate ultrasonic tones between 20000 and 24000 Hz, I 42 opted to increase the sample rate to 96000Hz to allow for more spectral distance between my clock signal and the audible spectrum. To start, I was most concerned wi th the rate of the clock, less so with the position. The clock needed to document the rate of time, but not necessarily what the time is. The clock position at this point is determined by the start of the clock as this becomes a common reference to a singl e instance across all recordings of the source. In future work, I endeavour to create a clock that both documents rate as well as a continuous refreshing position, such as longitudinal timecode (LTC) . In creating a clock to translate rat e , I began by cutti ng a continuous tone into regular intervals, creating pulses of tone. To start, the cycles of tone and silence were equal in length. This formed a binary of alternating states, tone on and tone off. It was quickly apparent that , despite creating the clock using inaudible ultrasonic tones, the clock created was indeed audible. The on/off cycles of ultrasonic frequencies created an emergent lower audible frequency. One of the first clocks I created was using a tone of 25 , 00 0 Hz. The clock was split between pul se s 1000 samples long and silence of equal duration. This duty cycle created a pulse wave that existed very much in the audible spectrum. These results held true despite changing the frequency of tone, the frequency of the clock , or the on/off duty cycle o f the pulses. There was still an emergent audible frequency in each of the clocks. A similar problem occurs when transforming signals from the time domain to the frequency domain through a fast Fourier transform. A range or window of time is selected to be translated by an FFT. This very selection, however, has an impact on the transformation. The type of window has a frequency 43 response that impacts the transformation between domains. The ‘windowing’ of my pulses created spectral artifacts just as they woul d in an FFT. Through softening the transition from tone to silence, therby removing the binary of states, the audible frequencies disappear red . While a most welcomed development, the pulses had lost some of their definition. The clock was no longer a binar y, but rather ultrasonic tones fading in and out. A downstream challenge would be in parsing a clock signal that did not have a clear edge between states. The first successful iteration of a clock was now complete, in that it could aurally translate a rate without being audible. This was a workable first clock iteration that would allow exploration of DSP alignment techniques. There would be many iterations to come as I developed the alignment system and gained feedback from the alignment system. 7.2 DSP Alignment System 7.2.1 A Dummy Clock The exploration of a DSP TOA alignment system began without sound. Ultimately, this system would take clock pulses, measure their rate, and correct for deviations fro m the reference rate. As such, these explorations began with methods to analyze a clock signal and correct it. Once I was confident in this system, I would connect it to the acoustically - viable clocking signal. My development start ed by creating a very simple clock system that could not be acoustically transmitted. This would act as a proxy for the eventual acoustically - viable clock. In creating arrays with on/off cycles I created my first rudimentary clock. The first iteration was an array with one positive value for every 100 indexes. The positive value was 1 and silence was zero . This effectively mimicked a very simple clock pulse with a frequency of 1/100 indexes. 44 Because of its binary nature, this clock was easily analyzed and the distance between each pulse could be easily extracted from the array. 7.2.2 Clock Distortion With this first clock in hand, a varispeed engine was created to affect the clock signal. Starting with a static varispeed, the rate of the clock pulses within the arra y was stretched through interpolation. For this process, nearest neighbour interpolation worked quite well, as I was not concern with the value of the clock, but more the position. By interpolating the array to 110% of its original length, I was left with clock pulses repeating every 110 indexes. However, as each index was stretched, there were now two clock pulses every 110 samples. Using a simple for loop , any index that was not the leading edge of a clock pulse was turned off, creating a cleaner array wi th 1 clock pulse value in every cycle. In shortening the array, compressing it to 80% of its length for example, some clock pulses were disappearing altogether. This was the result of the nearest - neighbour interpolation. Moving to linear interpolation solv ed this problem but required a more advanced loop to resolve the binary to the leading edge of the pulse. Only the first non - zero value in a pulse was promoted to full - strength, wherein the rest were demoted to a value of 0. With the clock data stretched a nd cleaned up to only document the leading edge of the clock pulses, the analysis of the array to extract the duration between clock pulses was straightforward and predictable. A varispeed of 110% resulted in a 1 clock pulse every 110 indexes. 7.2.3 Alignment Al gorithm The reference clock of 1/100 and stretched clocks provided the building blocks for a crude alignment system. The alignment would come from correcting the stretched clock back to the 1/100 frequency. 45 Both clocks started in sync, so their misalignmen t stems from their difference in rate. Correcting the rate would effectively correct their alignment. With a clock of 1/110, interpolating each cycle duration to 100 indexes would realign the clock with the reference. Using 110 as an example, dividing 110 by the reference value of 100 provides the indexes of interpolation. Resolving these values to an array of 100 samples completes the alignment. 7.2.4 Parallel Processing This abstract process of distorting and realigning clocks provide d a strong enough foundatio n to implement with real audio examples. The 1/100 clock was matched to a short audio clip. The clock was repeated so that its length matched the number of audio samples in the clip. These two arrays, clock and audio sample data, remained separate but now matched in terms of length. The same varispeed process was applied to both clock and audio sample data. Both arrays were temporally distorted by the same amount. After cleaning up the clock signal to identify the leading edge of the clock sample, the appli ed distortion and resultant clock cycle length was measured. Th e measurement of this clock signal was then applied to interpolate the audio data back to the reference clock. The distortions to the clock provided a map to restore the audio data back to its original temporality. Aiming to improve upon the process, methods of polynomial interpolation were applied to enhance the transparency of the process. The restoration of temporally distorted audio worked in a static sense. The audio data played back as if nothing happened. It wasn’t perfect, playing it back against an inverted copy of the original illustrated 46 some artifacts of the process. However, this phase check did reveal that there were indeed fewer artifacts in employing polynomial interpolation. 7.2.5 Modu lating Temporal Distortion The next challenge was to employ a modulating temporal distortion, known musically as vibrato, to the audio material and see if the same clocking method would hold up. Th is vibrato effect modulates the speed of the audio data ove r time. Using a sine function, the vibrato method continuous ly warps the audio at a specified rate and by a specified amount. The matched clock and audio data were subjected to the modulating time distortion and was able to be realigned with the reference audio. Audio processed with this effect sounds warped, alternately increasing and decreasing in pitch. The purpose of this temporal distortion was to probe the capabilities of my alignment methods. They had previously worked on static varisped audio, but i t was unknown if it would work with a fluctuating temporal distortion. While ultimately a developmental placeholder, the modulating pitch of this warped audio is analogous to moving microphones, albeit on a different scale. If a source and destination are moving toward one another, the cycles of compression and rarefaction emanating from the sound source are temporally compressed, resulting in a higher perceived pitch. The inverse is true when source and destination are diverging. The divergence results in expansion of the cycles of compression and rarefaction are lengthened , lowering their frequency. Thus, as microphones and sources are constantly moving their cycles of compression and rarefaction transduced by the microphone are either being compressed (pi tched up) or expanded (pitched down) depending on the movement. The synthesized vibrato created by these methods would act as a proof - of - concept source for aligning microphones. It’s important to note that he audible temporal and pitch distortions of my vi brato method are exaggerations of an order of magnitude beyond the temporal and pitch distortions created 47 by moving microphones. In reality, we only perceive these distortions on objects travelling at significant speed, the doppler effect of a passing trai n for example. The movement of microphones in a production context, do not register as doppler as the magnitude of movement of source and destinations is at an order of magnitude lower, therefore not registering as a perceivable doppler effect. Without mod ification to the alignment method, I was able to restore the warped audio to it’s original state. A continuous tone (sine wave) that I had warped was no longer pitching up and down; it was now completely flat. Music examples had the same dramatic results, I was unable to tell the original audio, free from temporal distortions, apart from the distort and realigned results. Further phase analysis did reveal artifacts of the process, but these artifacts were not apparent to me the listener without the phase analysis process. 7.2.6 Real Clocks Propelled forward by the positive realignment of warped audio, the project was ready to move to the next phase. So far, realignment was achieved through parallel temporal distortion of parallel clock and audio data arrays. The clock at this point was not integrated into the audio itself, rather it was a separate parallel entity. The goal of this development is to have the clock integrated into the audio itself, residing in the ultrasonic domain alongside the sonic content of the source. The parallel clock model was an effective proof - of - concept for DSP alignment, but it was unknown if these methods would work with an embedded ultrasonic clock. The parallel iteration of the clock was simple to analyze. In that work, the clock was a simple binary of values. The current iteration of clock analysis would prove to be more of a challenge as it’s complexity had evolved to be audio pulses in the ultrasonic domain. B efore merging the clock in - band with audio material, I endeavoured to analyze and extract a rate from the ultrasonic clock alone. 48 My first inclination to detect the clock signal was to use free Fourier transforms to look for activity in the frequency domai n to detect the frequency of the clock pulse. While I was indeed able to quantify activity of the clock in the frequency domain, I found it difficult to precisely identify the leading edge of the pulses. When translating from the time domain to the frequen cy domain, a window of samples is used to create the translation. As the window is the range of samples used in the translation, the results of the translation cannot be used directly to pinpoint a moment within that window. By comparing strengths of the c lock frequencies across different windows allowed for a rough approximation of the leading edge, but it was not precise enough to satisfy the requirements for the proposed time alignment application. Another challenge to this relatively rudimentary FFT app roach is the doppler shift in pitch caused by movement of the microphones. The reference clock has a fixed pitch that’s easy to identify. However, this pitch shifts a the microphone moves relative to the source, making precise analysis another variable to contend with in an FFT detection model. 7.2.7 Feature Detection Moving beyond this approach, I explored methods of feature extraction to identify the leading edge of each pulse. Librosa is an audio feature extraction library for the python scripting language and ‘ provides the building blocks necessary to create music information retrieval systems .’ 21 One of Librosa’s core features is onset detection, identifying the onset of notes or percussive transients in a piece of music. As the clock pulses are transients, su dden changes in sound pressure level, the intent was that Librosa would be able to detect their onset. However, as the pulses exist in the ultrasonic domain, they were largely ignored by the Librosa onset detection methods. As they were not audible, they w ere not registered by the system designed to detect audible musical events. However, cheating the sample rate 21 "LibROSA." LibROSA. Accessed March 19, 2019. https://librosa.github.io/librosa/. 49 of the clock audio to a lower rate, thereby making the pulses audible , allowed Librosa’s onset detection to perceive the pulses with a high degree of accuracy. The sample rate of the clock data was 96000Hz, allowing for the clock pulses to exist in the ultrasonic spectrum. However, instructing Librosa to process the audio as if it was sampled at 2400Hz dramatically reduced the pitch of the clock pul ses, bringing them back into the audible domain and allowing for their detection by Librosa. With this cheat, Librosa interpreted the ultrasonic clock and returned arrays documenting sample positions of the leading edge. The position was off by a number of samples, but consistently so. While off somewhat positionally, the rate of the clock was accurately resolved by the onset detection process. L owering the sample rate on input for Librosa’s onset detection method resulted in longer durations for interpret at ion and ultimately used vast amounts of memory that prevented the onset method from executing. Dividing the data into manageable chunks of 100,000 samples allowed the method to function. A method was created to separate the audio data into 100,000 sample blocks. The onset detection results were sequentially appended to arrays and the offset of the block added to the results. This created a transparent process wherein a large audio file could be processed. However, onsets at or near the start of the block were ignored and omitted from the results. As the current working clock had a cycle of 2000 samples, this occurred at every block. As a workaround, albeit a tempo ra ry one, the block size was altered to an interval that did not cycle sympathetically with th e clock . However, despite this fix there were still some pulses that aligned with the beginning of an onset detection cycle and subsequently missed by the scan. To resolve these gaps in the clock, a method was created to add a clock pulse where one was mis sing . For example, if a gap between detected pulses is significantly higher than the reference, the missing pulse is interpolated between the two detected pulses. 50 7.2.8 Clock with Audio With successful detection of the ultrasonic clock pulse, the next step was t o achieve similar results with the ultrasonic clock mixed in - band with standard audio data. The data used was my default musical example. Originally recorded at 44.1kHz, the material had a maximum frequency of 22050Hz resultant from the Nyquist theorem and confirmed with spectral analysis. The example clip was resampled to 96000Hz making its framerate compatible with the ultrasonic clock. Following this, a method was created to take a single cycle of the ultrasonic clock and repeat it to match the duration of the audio data . The clock and audio data arrays were then summed. I n their summation , both sources were attenuated to avoid overmodulation. With clock and audio data summed together, the resultant audio was exported and aurally inspected to ensure if th e clock was indeed inaudible , d ue its high frequency. The summation had worked, a visual spectral reading confirmed the presence of the clock in the ultrasonic domain which was completely inaudible in the audible spectrum. To see if the clock function as expected, t he next process was to isolate the clock and analyze it for positional and rate information. The audio would need to be separated to prevent the musical transients of the audio data from being detected as the clock. To isolate the clock from the audible sonic frequencies, the au dio data was filtered with a high - pass filter to pass the ultrasonic frequencies and cut the low frequencies. Interestingly, applying a high - pass filter to the ultrasonic clock had the same effect as fading the pulses in and out. By removing all frequencie s below 25000Hz with a 6 th order Butterworth high - pass filter, the net effect on the single frequency clock pulses was a short fade in and out of the pulses, softening their transition. Despite this, this process successfully separated the audible from the inaudible. Running this separated audio through the feature extraction algorithms provided the desired interval information, an array of 51 sample positions of the pulses’ leading edge. With the filter softening the pulses, the interpretation was offset slig htly. However, the overall distance or duration between pulses was intact. After successful analysis of the embedded clock on temporally neutral audio, the next test was to distort the temporality of the clocked audio and see if the clock remained useable. Applying vibrato to audio merged with the ultrasonic clock provided the expected results. The resultant distance between pulses expanded and contracted depending on the phase of the vibrato process. In order to realign the audio back to the reference, the audio had to be fed through the interpolation engine, using the timing information pulled from the ultrasonic clock. The interpolation realignment algorithm had to be redesigned, tailoring it to the cycles of the embedded ultrasonic clock signal, the refe rence distance between clock pulses. Following this modification, the audio was passed through the revised interpolation engine. The algorithm aligned the audio back to its original temporality. The vibrato ‘wow’ was reversed and the musical example is ret urned to something resembling its original state, no longer speeding up and slowing down. Th is correction was not as precise as the earlier parallel clock experimentations. The increased distance between clock pulses ultimately reduces the accuracy of the realignment algorithm. Whereas the parallel clock iteration restored a continuous tone to a single pitch, there are slight fluctuations present in the embedded clock experiments. These are most apparent with a continuous tone and less so with complex music mixes. 52 8 Conclusions and Future Work At this stage in its development, 2020 Sound restores the original temporality back to a temporally distorted audio file. This is a successful, albeit virtual, proof - of - concept t hat negates the movement of microphones in space. The prototype demonstrates that an in - band clocking signal can be used to both document an audio recording ’ s temporality and provide the roadmap to realign it back to the reference. Regardless of a microphone’s movement in space, its original rat e can be restored, reversing the minute f luctuation s in its pitch. This work is an important first step to realize the larger design goals of 2020 Sound with the intent of moving forward to create a practical toolset that would increase clarity, focus, and ultimately the impact possible in soundtrack media. In the virtual domain, work can be done to finesse the ultrasonic clock embedded alongside the audible material. Minimizing the distance between clock pulses would benefit the accuracy of the realignment algorithm. The clock pulses implemented in the scope of this project proves that this method is functional, but work must be done to demonstrate its v iability as an approach. The alignment algorithm itself is another vector for improvement in future iterations. The alignment is calculated solely between clo ck pulses. The rate of correction stairsteps from pulse to pulse, with the amount of rate - change varying in each cycle of interpolation. A spline or polynomial interpolation in the change of rate between clock pulses would effectively smooth this, continuo usly morphing the rate of interpolation between clock pulses. Interpolating the rate of interpolation would create a smooth curve of rate adjustment between clock pulses. This would reduce the artifacts produced by the relatively wide clock pulses applied in the current prototype . 53 The intention of the 2020 Sound design to align multiple microphone sources to one another demands that these explorations transition from the virtual to the acoustic. Moving into acoustic space will be a daunting endeavour and wi ll present many challenges to the full realization of the design goals. Emitting the ultrasonic clock pulse acoustically will undoubtedly present obstacles . The highly directional nature of high - frequency sound, for example, will impact the ability to succ essfully capture the reference clock pulses. By far, the biggest challenge is the potential impact that reflections and reverberations of clock pulses in an acoustic environment will have on their feasibility in a downstream alignment algorithm. The algori thm will have to be finessed to distinguish between direct clock pulses and reflected ones. As illustrated, the equipment used to transduce, amplify, route, and record the audio in production also adds uncertainty and challenges to future iterations. Microphones and recorders that can accurately capture and document the clock pulses will be a necessity. The prototy pe developed focuses on negating movement of microphones but does not address the initial offset of microphones in time and space. Future iterations will align these offsets at the start of the recordings before processing to counteract movement. Unique ul trasonic beacons , each with signature clock pulse s, could allow for multiple sources to be tracked in a scene . As most scenes involve multiple actors, this could be a necessity to the perceived success of future iterations. Variations in pitch, rate, or de tails in pulse composition could differentiate sources, allowing for multiple clocks to function together in single scenario. 54 Bibliography Mads Soegaard and Rikke Friis Dam. 2013. The Encyclopedia of Human - Computer Interaction , 2nd Ed. (2nd ed.). The Int eraction Design Foundation. “10MX.” Antelope Audio. Accessed November 13, 2018. https://en.antelopeaudio.com/products/10mx/. “Auto - Align Post: The Fast & Simple Way to Get Location Mics in Phase.” Sound Radix. Accessed November 2, 2018. https://www.soundradix.com/products/auto - align - post/. “Celemony | Capstan.” Accessed November 22, 2018. https://www.celemony.com/en/capstan. “Chirp.” Wikipedia , November 2, 2018. https://en.wikipedia .org/w/index.php?title=Chirp&oldid=866897234. Ciarkowski, Andrzej, Andrzej Czyzewski, Marek Dziubinski, Andrzej Kaczmarek, Bozena Kostek, Maciej Kulesza, and Przemyslaw Maziewski. “Methods for Detection and Removal of Parasitic Frequency Modulation in Audi o Recordings.” In Audio Engineering Society Conference: 26th International Conference: Audio Forensics in the Digital Age , 2005. http://www.aes.org/e - lib/browse.cfm?elib=13246. Davis, Gary, and Ralph Jones. The Sound Reinforcement Handbook . 2. ed., 2. prin ting. Milwaukee, Wis: Hal Leonard, 1990. “Dugan Automatic Microphone Mixers.” Accessed November 1, 2018. https://www.dandugan.com/products/. “Interpolation Methods in Scipy.” Modesto Mas | Blog, October 28, 2015. https://mmas.github.io/interpolation - scipy. Niemitalo, Olli. “Polynomial Interpolators for High - Quality Resampling of Oversampled Audio.” . . Introduction , n.d., 60. Peters, Nils. “Sweet [Re]Production: Developing Sound Spatialization Tools for Musical Applications with Emphasis on Sweet Spot and o ff - Center Perception,” n.d., 305. 55 Purcell, John. Dialogue Editing for Motion Pictures : A Guide to the Invisible Art . Routledge, 2013. https://doi.org/10.4324/9780203784570. Rumsey, Francis, and Tim McCormick. Sound and Recording: Applications and Theory . Oxford, UNITED KINGDOM: Taylor & Francis Group, 2014. http://ebookcentral.proquest.com/lib/ryerson/detail.action?docID=1638630. “ST 12 - 1:2014 - SMPTE Standard - Time and Control Code.” ST 12 - 1:2014 , February 2014, 1 – 41. https://doi.org/10.5594/SMPTE.ST12 - 1.2014. “VocALign PRO 4 - Overview - Synchro Arts.” Accessed November 18, 2018. https://www.synchroarts.com/products/vocalign - pro/overview. Wallaszkovits, Nadja, Tobias Hetzer, and Heinrich Pichler. “Drift and Wow Correct ion of Analogue Magnetic Tape Recordings in the Analogue Domain Using HF - Bias Signals.” In Audio Engineering Society Convention 136 , 2014. http://www.aes.org/e - lib/browse.cfm?elib=17189. Zafari, Faheem, Athanasios Gkelias, and Kin K. Leung. “A Survey of In door Localization Systems and Technologies.” CoRR abs/1709.01015 (2017). http://arxiv.org/abs/1709.01015. A. Gausepohl, Kimberly, Woodrow W. Winchester, Tonya L. Smith - Jackson, Brian M. Kleiner, and James D. Arthur. “A Conceptual Model for the Role of Storytelling in Design: Leveraging Narrative Inquiry in User - Centered Design (UCD).” Health and Technology 6, no. 2 (July 2016): 125 – 36. https://doi.org/10.1007/s12553 - 015 - 0123 - 1 . vi Appendi x Methods The following is an exploration of clocking components that allow for temporal documentation of a source in acoustic space is followed by methods of interpolation that allow for the alignment of moving microphone recordings. Clock signals In order to flatten the temporality of a recording, thereby counteracting its movement in space, a clocking signal recorded in production is necessary. This clocking signal provides a reference to the temporality of the production scene, providing data that creates a baseline for a flat temp oral rate. As per the solutions to correction wow and flutter, this baseline can then be used to flat any deviations that occur, in this case from microphones moving relative to a clock emitting at the intended source. Rate and Position In production audio , the simplest form of sync is the clapping slate. By recording the clapping slate, a position is documented across all recordings. Cameras document the visible clap whereas an audio recorder documents the audible clap. This allows for their synchronizatio n in the postproduction process through the simple alignment of the clap present on all recordings (audio and visual) to a single point in an editorial timeline. However, this singular clapping slate fails to define a rate. If the clocks between recorders are running at a different rate, the resultant recordings will drift from one another as one recording will be faster than another. The sampling rates between recorders may vary, but their ultimate connection to time must match if the recordings are to mai ntain sync. vii A clap before a take (top slate) and after (tail slate) can be used to derive a rudimentary rate based on the duration between both claps. However, this fails to account for any clocking deviations that occur between the reference claps. It wou ld be impractical to clap a slate during the action of a take as the aural and visual slate would impact the intended purpose of the capture itself. However, the more positional references available, the higher the resolution of the rate that can be derive d. It is a caveat that no two clocks, no matter their precision, will run in perfect sync. Two clocks will always drift apart. Only by sharing a common clock can two recorders maintain sync between them. As such, camera and audio equipment are often connec ted by base system signal clocks such as Tri - Level or Bi - Level video sync in contemporary cameras or DARS (Digital Audio Reference Signal), also known as word clock, in digital audio recorders. These signals are electronic pulses that create a clock with a relatively high resolution. In the case of word clock, the clock has a frequency that matches the recording rate of the device, 48,000 cycles per second for example. The same applies for video sync signals, with the clock aligning to the frame rate of the production, 24 frames per second as an example. By connecting clocks in production, the resultant recordings will have a shared connection to the original temporality of the scene. However, while devices have a common reference, this says nothing to the q uality of the devices’ temporal connection to time itself. Rather, the imperfections of the master clock will be translated to all slave devices. While audio equipment can be connected to the output of a atomic clock 22 , most practical clocks are less exact ing. LTC The system level clocks of video and audio systems exists only to define a rate between devices, allowing them to run in sync with one another. The systems fails, however, to translate a positional reference to time. Their frequency describes the rate of time but cannot describe what time it is. 22 “10MX,” Antelope Audio, accessed November 21, 2018, https://en.antelopeaudio.com/products/10mx/. viii Longitudinal timecode (LTC) allows for the translation of both the rate and position of time. LTC is a Manchester encoded biphase signal that allows SMPTE timecode (Hour, Minute, Second, Frame) to be docume nted with each frame. The phase of each pulse translates bits of data that when decoded describe the position of time. A rate emerges from the continuous burst of positional references. The frequency of LTC resides in the audible frequency spectrum allowin g it to be distributed electronically and recorded by standard audio equipment. While the signal can be leveraged to drive the clock of a recorder it can also be recorded as a standard audio source. In this context, the original temporality of the clock ca n be restored regardless of the rate or variations in the recorders clock. Despite its utility in documentation the position and rate of time, the audible nature of LTC makes it unusable to track the relative position of microphones in space. Figure 15 - Manchester biphase encoding 23 Ultrasonic Chirps A clocking signal above the audible spectrum would allow for such a signal to exist acoustically without being audible. Human hearing tops out at 20,000 Hz but varies from individual t o individual. Anything above this 20kHz is above the range of human hearing. While outside of the range of human hearing, a wide range of audio equipment is able to capture above 20,000 Hz successfully. Contemporary professional audio recording equipment i s able to record with sampling rates as high as 192,000 samples 23 “ST 12 - 1:2014 - SMPTE Standard - Time and Control Code,” ST 12 - 1:2014 , February 2014, 1 – 41, https://doi.org/10.5594/SMPTE.ST12 - 1.2014. ix per second, allowing them to capture frequencies that max out at 96,000 Hz. This allows for a significant bandwidth of recordable sound that is above the human hearing threshold. Ultrasonic ch irps are a common method for ultrasonic ranging and localization that use short bursts of inaudible ultrasound to transmit a clock pulse. The chirps are defined by their modulation of frequency (FM). The design of the chirp can have the frequency sweep fro m one value to another over the length of the chirp. Figure 16 - Frequency modulated chirp sweeping up in frequency 24 Positional localization methods implement these chirps measure the time - of - arrival of a source at multiple perspe ctives, a microphone array. 25 The inverse is also a valid approach, to measure the time - of - arrival of multiple unique chirping sources at a single microphone. Regardless of approach, the time - difference - of - arrival (TDOA) allows for the calculation of the re lative location of an object in space. Whichever microphone has the earliest TOA is closest to the source, whichever has the last is furthest. A one - dimensional array can locate an object’s position between microphones. A two - dimensional array can locate a n object on a single plane. A three - dimensional array adds height to the equation allowing for the calculation of a location in three - dimensional space. 24 “Chirp,” Wikipedia , November 2, 2018, https://en.wikipedia.org/w/index.php?title=Chirp&oldid=866897234. 25 Faheem Zafari, Athanasios Gkelias, and Kin K. Leung, “A Survey of Indoor Localization Systems and Technologies,” CoRR abs/1709.01015 (2017), http://arxiv.org/abs/1709.01015. x Figure 17 - Time of Arrival positional localization 26 Interpolation To nullify the Doppler effect created by movement of either source or microphone, the inverse movement is applied the recordings. As source and microphone converge, slowing the recording has the potential to cancel the recorded movement. Conv ersely, divergence is counteracted by an increase in playback rate. In order to fix this varispeed playback to a new recording, the playback output is fed a recorder operation with a consistent clock speed. As an analog example, the output of a varisped pl ayback machine is connected to the input of a temporally consistent recorder. Fixing varispeed in a digital context applies the same principles through interpolation. As a digital recording is a sampling of amplitude at fixed intervals of time, techniques of interpolation can be implemented to change the playback rate of audio while maintaining the technical sampling rate of the audio file. Interpolation allows for the creation of new data points between existing data points in a set. For example, through i nterpolation a playback rate of 46,000 samples per second can be translated to a file with a standard rate of 48,000 samples per second. Through this rate change, new samples are created from the existing data set to effectively fill in the gaps between th e existing samples. This allows for a change in playback rate in a digital context. 26 Zafari, Gkelias, and Leung. xi Zero Order Hold and Nearest Neighbour The simplest forms of interpolation does not calculate new data points from the existing set. In zero order hold interpolation, an int erpolated data point equals the value of the previous fixed data point. Nearest neighbor creates new data points that equal the nearest fixed data, regardless if it falls before or after. While these techniques do allow for a change in audio playback rate, they create audible artifacts. Linear Interpolation Linear interpolation goes beyond these techniques by formulating a straight line between existing data points. Data points interpolated linearly exist on the line drawn between the two points. This drama tically improves the audible quality of interpolated audio data beyond nearest neighbor or zero order hold techniques. Polynomial Interpolations Polynomial interpolation considers multiple points (if not all) in a data set to define a curve, or more specif ically, a spline between existing points. However, a polynomial that considers too many data points is susceptible to Runge’s Phenomenon wherein increasing the number of points calculated through polynomials generates errors in interpolation. Limiting the quantity, or order, of fixed points used to generate interpolated points can avoid this phenomenon. Figure 18 - Zero Order Hold, Linear, Polynomial Interpolation 27 27 Olli Niemitalo, “Polynomial Interpolators for High - Quality Resampling of Oversampled Audio,” . . Introduction , n.d., 60. xii Modesto Mas’s blog post on Python polynomial implementation br eaks down polynomial interpolation in Python programming language using the SciPy libraries. SciPy is a library of mathematics, science, and engineering resources for Python which includes a wealth of functions that facilitation a range of methods for int erpolating data. Modesto’s blog connects the theory of polynomial interpolation with its execution. Feature Extraction In the context of this work, audio feature extraction provides purchase to detect sonic clock signals, thereby allowing for their interpo lation back to a reference. In an audio context, feature extraction tools analyze audio in order to identify features. This could be transients, the sudden change in sound pressure level indicative of a percussive hit. A feature extractor with an onset det ection algorithm identifies these transients within a recording and reports their positions. Another common feature extracted using these tool is frequency or pitch extraction. Through this, a feature extractor can identify notes in a musical composition. As an onset detector provides purchase to identify the leading edge of transients, it could be used to determine the leading edge of an ultrasonic clock pulse. However, as to not confuse the algorithm with audible sonic information that may include transie nts, the ultrasonic clock would need to be filtered from the rest of the audio spectrum. This filtering isolates the clock from the rest of the spectrum, in hopes of only leaving the clock intact for detection. 28 “Interpolation Methods in Scipy,” Modesto Mas | Blog, October 28, 2015, https://mmas.github.io/interpolation - scipy. Acoustic Virtual Future Work Conclusions After Before After Before SUCCESS! Feature Detection Transient Transient Transient Real Clocks Modulating Temporal Distortion TIME RATE/SPEED Parallel Processing 100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000 Alignment Algorithm interpolation Clock Distortion A Dummy Clock 100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000 DSP Alignment System Ultrasonic Clocking Prototyping Results User Interface Waveform B Waveform A Delta 2020 Static Dynamic Align Render User & Process Flow Output in-place audio export Render? Align to arrest movement Offset audio to common reference Align Sync Points Select Sync Point Dynamic ? Static? Align? Display Δ Validate Clock Display Waveform Load Select Files or Ranges Internal Process User Input User Narrative Design After receiving a locked cut of a film, Krystin begins the dialogue editorial process. As usual, she cuts dialogue from the source ISO recordings to sync with picture and cuts in alternate takes as required. After the production dialogue is edited, patched & finessed, Krystin launches the 2020 sound offline processor. Krystin selects multiple clips that belong to the same temporal recording set. She clicks 'Analyze’ and 2020 begins scanning the selected clips for a valid ultrasonic positional clock signal. If valid, Krystin can select which source she wants to align to. For clips with multiple actors, Krystin assigns body mics directly to a beacon. Depending on who’s talking at any given point, 2020 dynamically changes its alignment to focus on the actor currently speaking. Krystin, previews the alignment, tweaking the fine tuning settings as needed. Satisfied with the alignment, Krystin clicks 'Commit’, rendering aligned audio to new clips on a new editorial playlist. Jan is a production sound mixer for narrative & documentary film and television productions. She owns and maintains a recording kit that includes lavaliere, shotgun, and small diaphragm condenser microphones. These microphones track subjects in a scene as body, boom, planted, and boundary microphones. In addition to affixing body mics to her subjects, she discreetly places the 2020 Sound localization beacons to each subject she wants to track. She double checks that each beacon is on and has a unique address assigned to distinguish each beacon. Jan spec’d her microphones, wireless transmitters, and recorder to ensure they can record the ultrasonic clock pulses of the beacons. Jan records ISOs and mixes of each scene as per usual, delivering them to the production for use in offline picture editorial and dialogue editorial. Post Production Production Post Production Recording Reference Clock Alignment clock analysis Output interpolation POST PRODUCTION RECORDING ALIGNMENT Production Ultrasonic Beacon Microphone moving apart moving towards Reference Clock Recording PRODUCTION AUDIO TEMPORAL DOCUMENTATION Conceptual Framework The Design Sound Radix Auto Align Post Syncro Arts VocAlign Dan Dugan Automixer Celemony Capstan Existing Designs Learning by Teaching Iterative Prototyping Iterative Loop Large Problem Solution Testing Analyze & Reflect Small Problem Small Integrated Solution Test Analyze & Reflect Iterative Loop Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Small Problem Small Integrated Solution Test Analyze & Reflect Iterative Loop Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Small Problem Small Integrated Solution Test Analyze & Reflect Iterative Loop Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Small Problem Small Integrated Solution Test Analyze & Reflect Iterative Loop Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Small Problem Small Integrated Solution Test Analyze & Reflect Iterative Loop Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Small Problem Small Integrated Solution Test Analyze & Reflect Iterative Loop Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Xsmall Problem Sketch Gather Resources Prepare Environment Develop Test Analyze & Reflect Iterative Loop Solution Research Through Design Design of Research Methods for Design Design Design Research ResearchThroughDesign ResearchThroughDesign Design of Products Narrative-Driven User-Centred Design Overview Analogous Problems Interpolation Time of Arrival Feature Extraction Wow and Flutter Audio Production Localization Techniques Comparable Designs Auto Align Post Current Practices Production FFT Varispeed Techniques Clocking Techniques Post production TDOA Nearest Neighbour Polynomial Linear MELS DFT LTC Narratives of TOA Production Post production Vocalign Solutions Dan Dugan Automixer HF Bias Spectral Analysis Machine Learning Capstan Contextual Review - Research Materials Map Methodology Audio System Latency Music Recording Analog Wow & Flutter Current Alignment Methods Mix + = Editorial Film Production Audio Scenario plant boom lav boom lav plant Recordings Recordings Microphones Microphones Analogous Problems Post Production Production Time of Arrival Narratives Spatial Audio SOURCE shorter longer right ear left ear Comb Filtering Summing multiple perspectives + = Mixes lack clarityfocussharpnessprecision Reflections SOURCE DIRECT/SHORTHER DESTINATION REFLECTED/LONGER Time of Arrival SOURCE TIME/DISTANCE EARLIER/NEARERDESTINATION LATER/FURTHERDESTINATION LONGER SHORTER earlier/nearer later/further EARLIER/NEARER LATER/FURTHER Frequency and Wavelength frequency = speed of sound/wavelengthwavelength = speed of sound/frequency time domain 1 cycle time frequency frequency domain Sound Fundamentals 343 Metres per Second compression rarefaction time Properties of Sound How? Why? What? Introduction 2020 Sound is a positionally-aware microphone and DSP time-of-arrival alignment system. 2020 Sound is both a production positional tracking tool and post production alignment process. In production, an ultrasonic beacon emits a temporal positional reference that is captured by standard audio recording devices. This reference gives purchase to align multiple microphone perspectives of a source, correcting for their initial offset as well their movement throughout the recording. In capturing a sound source with multiple microphones, misaligned and drifting time-of-arrival of the source at each microphone greatly impacts the cohesion, focus, and impact of their summation in the mixing process. The common boom and lavaliere microphone scenario implemented in film and television production suffers from this misalignment and, as a result, time-intensive and inaccurate manual editorial processes are employed to align microphones before their summation. This system could remedy a fundamental issue encountered in audio production with the ultimate aim of improving the clarity and quality of productions that make use of 2020 Sound. Abstract 2020 Sound How To